novox/mesh-delivery
19
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove the lease write, take-over and their tools from mesh-grants (issue 1126, B7a)

Browse Source 
The controller ended by its unassignment (ADR 0590), which supersedes the
lease write and its hand-back whole and makes the grants part's list the
only one the bus node follows. The lease write, its check, the removal
marker, the take-over and hand-back asks, their five tools, the seat
claim and the lease and takeover states had nothing left to act on; the
membership notice on its subject had no reader. A whole-module test on
the bus replaces the lease write's: with no controller at all, the part
publishes a membership in its state alone, makes the module's consumer
and writes the key the bus node follows.

Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
This commit is contained in:
jschoubben
2026-11-02 11:25:59 +01:00
parent be88162cde
commit 14046f9b14
26 changed files with 373 additions and 3898 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/mesh-grants/cmd/mesh-grants/asking.go
+2 -78
View File
@@ -2,77 +2,14 @@ package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// The three bus adapters for mesh-sdk go/asker on the part's own credential, as mesh-delivery-teller has them (copied: the
// SDK's asker reaches the bus through a bundle's runtime; moving the adapters into the SDK is a change of its own). What
// the account may do is the controller's composed grant for a module that uses the seat operator-channel: publish its own
// ask and condition, hear its own decided event, and read the router's record of its own asks.
const recordWithin = 30 * time.Second
// messengerAsks is the router's records bucket as the seat operator-channel's manifest declares it (its `records`), the
// one the controller composes this account's direct get on (mesh-sdk buscompose SeatTrafficOf): never `asks`, a bucket
// no grant names (issue 774). The SDK names no constant for it; mesh-mail and pacman keep their own the same way.
const messengerAsks = "messenger_asks"
func publishVia(ctx context.Context, js jetstream.JetStream) func(subject string, body any, id string) error {
return func(subject string, body any, id string) error {
data, err := json.Marshal(body)
if err != nil {
return err
}
var opts []jetstream.PublishOpt
if id != "" {
opts = append(opts, jetstream.WithMsgID(id))
}
_, err = js.Publish(ctx, subject, data, opts...)
return err
}
}
func recordVia(ctx context.Context, conn *nats.Conn, module string) func(id string) (*stdio.StateEntry, error) {
return func(id string) (*stdio.StateEntry, error) {
ctx, cancel := context.WithTimeout(ctx, recordWithin)
defer cancel()
msg, err := conn.RequestWithContext(ctx, "$JS.API.DIRECT.GET.KV_"+messengerAsks+".$KV."+messengerAsks+"."+module+"."+id, nil)
// No responders is the bucket out of reach for a moment, never "no record": read as none, a live ask would be
// closed by Reconcile a minute later and the operator's yes lost.
if errors.Is(err, nats.ErrNoResponders) {
return nil, fmt.Errorf("the router's record of %s cannot be reached: %w", id, err)
}
if err != nil {
return nil, err
}
if status := msg.Header.Get("Status"); status == "404" || len(msg.Data) == 0 {
return nil, nil
}
return &stdio.StateEntry{Key: id, Value: msg.Data}, nil
}
}
func decidedVia(conn *nats.Conn, module string, logf func(string, ...any)) func(pattern string, handle func(stdio.Envelope) error) error {
return func(pattern string, handle func(stdio.Envelope) error) error {
_, err := conn.Subscribe(asks.DecidedSubject(module), func(m *nats.Msg) {
if err := handle(stdio.Envelope{Key: pattern, Body: m.Data}); err != nil {
logf("a warrant could not be handled and is read again from the router's record at the next start: %v", err)
}
})
return err
}
}
// sayVia says a condition of the part on its own subject, as a module that uses the seat operator-channel may: the
// account consumers that differ, a grant missing, a user list held, a principal refused.
func sayVia(ctx context.Context, js jetstream.JetStream, module string) func(c asks.Condition) error {
return func(c asks.Condition) error {
subject, err := asks.ConditionSubject(module)
@@ -87,16 +24,3 @@ func sayVia(ctx context.Context, js jetstream.JetStream, module string) func(c a
return err
}
}
// seatSaying says the seat mesh-grants' events on its seat subject into EVENTS, as the module that holds it (novox/hq
// ADR 0582 clauses 3, 4, 6): lease-handed, lease-altered and removal-failed.
func seatSaying(ctx context.Context, js jetstream.JetStream) func(event string, body any) error {
return func(event string, body any) error {
data, err := json.Marshal(body)
if err != nil {
return err
}
_, err = js.Publish(ctx, "mesh.seat."+catalogue.GrantsSeat+".event."+event, data)
return err
}
}
modules/mesh-grants/cmd/mesh-grants/integration_test.go
+123 -237
View File
@@ -6,12 +6,11 @@ import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"slices"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
@@ -19,147 +18,21 @@ import (
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/asker"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-sdk/go/membership"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/seal"
"mesh-grants/internal/grants"
)
// standIns are the seats and tools the lease write asks, standing in on the real bus's side: node-service-manager on
// the controller's node (a unit's ActiveState, as the systemd module answers it), the bus module's expiry tool acting on
// the real lease bucket's stream (as nats-tools' SetBucketExpiry does), and the checks answering green.
type standIns struct {
ctx context.Context
js jetstream.JetStream
mu sync.Mutex
active string
calls []string
}
func (s *standIns) Call(key string, body any) (json.RawMessage, error) {
s.mu.Lock()
s.calls = append(s.calls, key)
s.mu.Unlock()
base, _, _ := strings.Cut(key, "@")
args, _ := body.(map[string]any)
var out any
switch base {
case grants.CallServiceStop:
s.active, out = "inactive", map[string]any{"ok": true}
case grants.CallServiceStart:
s.active, out = "active", map[string]any{"ok": true}
case grants.CallServiceStatus:
out = map[string]any{"ActiveState": s.active}
case grants.CallBucketExpiry:
now := fmt.Sprint(args["max_age"])
d, err := time.ParseDuration(strings.Replace(now, "none", "0s", 1))
if err != nil {
return nil, err
}
st, err := s.js.Stream(s.ctx, "KV_"+fmt.Sprint(args["bucket"]))
if err != nil {
return nil, err
}
cfg := st.CachedInfo().Config
cfg.MaxAge = d
if d > 0 && (cfg.Duplicates == 0 || cfg.Duplicates > d) {
cfg.Duplicates = d
}
if _, err := s.js.UpdateStream(s.ctx, cfg); err != nil {
return nil, err
}
out = map[string]any{"bucket": args["bucket"], "now": now, "changed": true}
case grants.CallIssue:
out = map[string]any{"status": "resolved"}
case grants.CallNodeTrusts:
out = map[string]any{"every": true, "not_yet": []string{}}
case grants.CallConfigCompare:
out = map[string]any{"source": "indexer", "count": 0}
case grants.CallUpgrade:
out = map[string]any{"empty": true, "splits": []any{}}
case grants.CallVaultStatus:
out = map[string]any{"ready": true, "minting": map[string]any{"on": true, "held_back": map[string]string{}}}
case grants.CallStreams:
out = map[string]any{"detail": []any{}}
case grants.CallUsers:
out = map[string]any{"users": []any{}}
case grants.CallRecordWarrant, grants.CallRecordHandAct:
out = map[string]any{}
default:
return nil, fmt.Errorf("no stand-in for %s", key)
}
return json.Marshal(out)
}
// memStore is the part's own state, in memory, with its time to live per key recorded.
type memStore struct {
mu sync.Mutex
vals map[string][]byte
revs map[string]uint64
}
func (m *memStore) Get(state, key string) ([]byte, uint64, bool, error) {
m.mu.Lock()
defer m.mu.Unlock()
v, ok := m.vals[state+"/"+key]
return v, m.revs[state+"/"+key], ok, nil
}
func (m *memStore) Put(state, key string, value []byte) (uint64, error) {
m.mu.Lock()
defer m.mu.Unlock()
m.vals[state+"/"+key] = append([]byte(nil), value...)
m.revs[state+"/"+key]++
return m.revs[state+"/"+key], nil
}
func (m *memStore) PutTTL(state, key string, value []byte, _ time.Duration) (uint64, error) {
return m.Put(state, key, value)
}
func (m *memStore) Keys(state string) ([]string, error) {
m.mu.Lock()
defer m.mu.Unlock()
var out []string
for k := range m.vals {
if s, key, _ := strings.Cut(k, "/"); s == state {
out = append(out, key)
}
}
return out, nil
}
func (m *memStore) Delete(state, key string) error {
m.mu.Lock()
defer m.mu.Unlock()
delete(m.vals, state+"/"+key)
return nil
}
type memObjects struct{}
func (memObjects) PutObject(string, []byte) error { return nil }
func (memObjects) DeleteObject(string) error { return nil }
// yesAsker answers every ask the part raises with the operator's yes, as the router's warrant hands it.
type yesAsker struct{ asked []string }
func (a *yesAsker) Ask(q asks.Ask) error { a.asked = append(a.asked, q.ID); return nil }
func (a *yesAsker) Where(string) (string, *asks.Warrant, error) { return "open", nil, nil }
func (a *yesAsker) Cancel(string) error { return nil }
func (a *yesAsker) Take(w asks.Warrant, act func(asks.Option, asks.Warrant) error) (asker.Handled, error) {
return asker.Acted, act(asks.Option{ID: w.Option}, w)
}
// Issue 774, 774/S23 (novox/hq ADR 0582 clauses 3, 5, "How it is checked"): mesh-grants' lease write and hand-back
// against the real bus, with a stand-in controller holding the lease in a bucket with an expiry (2 s standing in for 15
// s, so the test waits past it in seconds), and stand-in node-service-manager and bus tool seats.
func TestTheLeaseWriteOnTheBus(t *testing.T) {
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "lease")
// TestTheGrantsPartWhole starts the program the way the node does (its entry, run, with its environment and a broker
// file) against an embedded bus that holds no controller at all: no lease bucket, no controller placed (novox/hq ADR 0590,
// issue 1126). Given a module placed with an event it consumes, it publishes the module's membership in its own state and
// on no subject, makes the module's durable consumer on EVENTS, and writes the sealed list's key for the bus node to follow
// alone; and it creates no lease, nor any state of a lease write or a take-over.
func TestTheGrantsPartWhole(t *testing.T) {
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "grants")
if err != nil {
t.Fatal(err)
}
@@ -182,128 +55,141 @@ func TestTheLeaseWriteOnTheBus(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
const ttl = 2 * time.Second
was := grants.LeaseExpiry
grants.LeaseExpiry = ttl
t.Cleanup(func() { grants.LeaseExpiry = was })
lease, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: buscompose.LeaseBucket, History: 1, TTL: ttl})
if err != nil {
t.Fatal(err)
}
controller, _ := json.Marshal(buscompose.LeaseHolder{Instance: "controller@box-a pid 1", Host: "box-a", Epoch: 60})
if _, err := lease.Create(ctx, buscompose.ControllerLeaseKey, controller); err != nil {
t.Fatal(err)
}
bucket := func(state string) jetstream.KeyValue {
kvs := map[string]jetstream.KeyValue{}
for _, state := range grants.Reads {
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: bucketOf(state), History: 5})
if err != nil {
t.Fatal(err)
}
return kv
kvs[state] = kv
}
put := func(kv jetstream.KeyValue, key string, v any) {
for _, state := range grants.States {
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: grants.Module + "_" + state, History: 5})
if err != nil {
t.Fatal(err)
}
kvs[grants.Module+"."+state] = kv
}
for _, c := range []jetstream.StreamConfig{
{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}},
{Name: "CONDITIONS", Subjects: []string{"mesh.seat.operator-channel.accept.condition.>"}},
} {
if _, err := js.CreateStream(ctx, c); err != nil {
t.Fatal(err)
}
}
put := func(state, key string, v any) {
t.Helper()
raw, err := json.Marshal(v)
if err != nil {
t.Fatal(err)
}
if _, err := kv.Put(ctx, key, raw); err != nil {
if _, err := kvs[state].Put(ctx, key, raw); err != nil {
t.Fatal(err)
}
}
nats := `{"module":"nats","version":"1","claims":[{"name":"mesh-broker"}],"own-secrets":{"broker":"/var/lib/nats/broker"},` +
`"tools":["nats_bucket_expiry","nats_consumer_remove","nats_stream_remove"]}`
digest := delivery.ManifestDigest([]byte(nats))
put(bucket(grants.ReadManifests), digest, delivery.Manifest{V: delivery.Version, Module: "nats", JSON: nats})
put(bucket(grants.ReadModules), "nats", delivery.Module{V: delivery.Version, FP2: "src2:" + strings.Repeat("ab", 32), Manifest: digest,
Repository: "novox/mesh-catalog", Commit: "c"})
pl, _ := delivery.EncodePlacement(delivery.Placement{Node: "a", Modules: []delivery.PlacedModule{{Module: "nats"}, {Module: grants.Controller}}})
if _, err := bucket(grants.ReadPlacement).Put(ctx, "placement.a", pl); err != nil {
manifest := func(module, raw string) {
digest := delivery.ManifestDigest([]byte(raw))
put(grants.ReadManifests, digest, delivery.Manifest{V: delivery.Version, Module: module, JSON: raw})
put(grants.ReadModules, module, delivery.Module{V: delivery.Version, FP2: "src2:" + strings.Repeat("ab", 32), Manifest: digest,
Repository: "novox/x", Commit: "c"})
}
manifest("nats", `{"module":"nats","version":"1","claims":[{"name":"mesh-broker"}],"own-secrets":{"broker":"/var/lib/nats/broker"}}`)
manifest("mod-c", `{"module":"mod-c","version":"1","own-secrets":{"broker":"/var/lib/c/broker"},"consumes":["nats.done"]}`)
pl, _ := delivery.EncodePlacement(delivery.Placement{Node: "a", Modules: []delivery.PlacedModule{{Module: "nats"}, {Module: "mod-c"}}})
if _, err := kvs[grants.ReadPlacement].Put(ctx, "placement.a", pl); err != nil {
t.Fatal(err)
}
put(bucket(grants.ReadTakeover), health.TakeoverKey, health.Takeover{V: 1, Follow: health.FollowKeeper, By: "operator", Why: "taken"})
put(bucket(grants.ReadAccounts), buscompose.ControllerName, map[string]any{"hash": "$2a$11$controller", "generation": 1, "at": "2026-11-01T00:00:00Z"})
busKey, _ := seal.GenerateKeyPair()
have := bucket(grants.ReadHave)
loads := func(source, digest string) {
put(have, "a.nats", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK,
UserList: &delivery.UserListWritten{Source: source, Revision: 1, Digest: digest, Users: 1, Key: busKey.PublicKey}})
}
loads("a.nats.src2", "sha256:aa")
put(grants.ReadHave, "a.node", map[string]any{"v": 1})
put(grants.ReadHave, "a.nats", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK,
UserList: &delivery.UserListWritten{Source: delivery.GrantsUserListSource, Revision: 1, Digest: "sha256:aa", Users: 1,
Key: busKey.PublicKey, Follows: delivery.FollowGrants}})
seats := &standIns{ctx: ctx, js: js, active: "active"}
store := &memStore{vals: map[string][]byte{}, revs: map[string]uint64{}}
p := &grants.Part{Read: newBusReader(ctx, js), Call: seats, Store: store, Lease: &leaseHand{ctx: ctx, js: js},
Asker: &yesAsker{}, Objects: memObjects{}, Nonce: func() string { return "0001" }}
act := func(ask func(string, string) (any, error)) {
t.Helper()
if _, err := ask("operator@shanks", "issue 774's integration test"); err != nil {
t.Fatal(err)
}
p.Compose() // the process raises the ask
r, _ := p.LeaseState()
if r.Open == nil {
t.Fatalf("no act open: %+v", r)
}
if err := p.Warranted(asks.Option{ID: "yes"}, asks.Warrant{Ask: r.Open.Ask, Option: "yes", Outcome: asks.OutcomeChosen}); err != nil {
t.Fatal(err)
}
r, _ = p.LeaseState()
if r.Open == nil || r.Open.List == "" {
t.Fatalf("the act is not waiting on its list: %+v", r)
}
loads(delivery.GrantsUserListSource, r.Open.List) // the bus node loads it
p.Compose()
if r, _ := p.LeaseState(); r.Open != nil {
t.Fatalf("the act did not finish: %+v", r.Open)
}
// The bus module's read tools, as nats-tools answers them: the streams with their consumers as the bus holds them now.
answer := func(m *nats.Msg, result any) {
raw, _ := json.Marshal(map[string]any{"result": result, "node": "a"})
_ = m.Respond(raw)
}
expiry := func() time.Duration {
st, err := lease.Status(ctx)
if err != nil {
t.Fatal(err)
sub, err := nc.Subscribe("mesh.mod.nats.>", func(m *nats.Msg) {
switch {
case strings.Contains(m.Subject, ".nats_users"):
answer(m, map[string]any{"users": []any{}})
case strings.Contains(m.Subject, ".nats_streams"):
var args struct{ Stream string }
_ = json.Unmarshal(m.Data, &args)
detail := []any{}
if st, err := js.Stream(ctx, args.Stream); err == nil {
consumers := []any{}
names := st.ConsumerNames(ctx)
for name := range names.Name() {
c, _ := st.Consumer(ctx, name)
cfg := c.CachedInfo().Config
consumers = append(consumers, map[string]any{"name": name, "filters": cfg.FilterSubjects, "max_deliver": cfg.MaxDeliver})
}
detail = append(detail, map[string]any{"name": args.Stream, "consumer_detail": consumers})
}
answer(m, map[string]any{"detail": detail})
default:
answer(m, map[string]any{})
}
return st.TTL()
}
// Issue 774: the operator's answer is read from the router's own bucket, the one the controller grants (the seat's
// records, messenger_asks), as the process reads it.
put(bucket(messengerAsks), "mesh-grants.lease-write-0001", map[string]any{"status": "decided"})
if rec, err := recordVia(ctx, nc, "mesh-grants")("lease-write-0001"); err != nil || rec == nil {
t.Fatalf("the router's record of the ask: %v, %v; want it read", rec, err)
}
// 774/S23/AC1.
act(p.LeaseWrite)
e, err := lease.Get(ctx, buscompose.ControllerLeaseKey)
})
if err != nil {
t.Fatal(err)
}
if away, says := buscompose.LeaseHandedAway(e.Value(), true, nil); !away || !strings.Contains(string(e.Value()), `"handed-to":"mesh-grants"`) {
t.Fatalf("the lease after the write: %s (%s)", e.Value(), says)
}
if expiry() != 0 || seats.active != "inactive" {
t.Fatalf("the lease's expiry is %s, the controller %s", expiry(), seats.active)
}
// Issue 774: the stop goes to the node placement places the controller on, not the lease's host.
if !slices.Contains(seats.calls, grants.CallServiceStop+"@a") || slices.Contains(seats.calls, grants.CallServiceStop+"@box-a") {
t.Fatalf("the stops asked: %v, want one on a", seats.calls)
}
if _, err := lease.Create(ctx, buscompose.ControllerLeaseKey, controller); !errors.Is(err, jetstream.ErrKeyExists) {
t.Fatalf("the controller's retake: %v, want refused", err)
}
time.Sleep(ttl + time.Second)
if e, err := lease.Get(ctx, buscompose.ControllerLeaseKey); err != nil {
t.Fatalf("the lease is gone past the old expiry: %v", err)
} else if away, _ := buscompose.LeaseHandedAway(e.Value(), true, nil); !away {
t.Fatalf("the lease past the old expiry: %s", e.Value())
t.Cleanup(func() { _ = sub.Unsubscribe() })
// A membership is never said on its subject any more (issue 1126).
var said atomic.Int32
if _, err := nc.Subscribe("mesh.assignment.>", func(*nats.Msg) { said.Add(1) }); err != nil {
t.Fatal(err)
}
// 774/S23/AC2.
act(p.LeaseHandBack)
if expiry() != ttl || seats.active != "active" {
t.Fatalf("after the hand-back the expiry is %s, the controller %s", expiry(), seats.active)
broker := filepath.Join(dir, "broker")
cred, _ := json.Marshal(process.Account{URL: srv.ClientURL(), Node: "a", Module: grants.Module, User: "a." + grants.Module, Password: "pw"})
if err := os.WriteFile(broker, cred, 0o600); err != nil {
t.Fatal(err)
}
if _, err := lease.Create(ctx, buscompose.ControllerLeaseKey, controller); err != nil {
t.Fatalf("the controller cannot take the lease back: %v", err)
env := map[string]string{"MESH_BROKER_FILE": broker}
runCtx, stop := context.WithCancel(ctx)
done := make(chan error, 1)
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
t.Cleanup(func() { stop(); <-done })
until := func(what string, ok func() bool) {
t.Helper()
for deadline := time.Now().Add(30 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if ok() {
return
}
}
t.Fatalf("never saw: %s", what)
}
until("a.mod-c's membership in the part's own state memberships", func() bool {
e, err := kvs[grants.Module+"."+grants.StateMemberships].Get(ctx, "a.mod-c")
if err != nil {
return false
}
m, err := membership.DecodeGrantsMembership(e.Value())
return err == nil && m.Module == "mod-c"
})
until("mod-c's durable consumer on EVENTS, made by the part", func() bool {
_, err := js.Consumer(ctx, "EVENTS", "a_mod-c")
return err == nil
})
until("the sealed list's key, followed by the bus node alone", func() bool {
e, err := kvs[grants.Module+"."+grants.StateUserList].Get(ctx, grants.KeySealed)
if err != nil {
return false
}
g, err := delivery.DecodeGrantsUserList(e.Value())
return err == nil && g.Follow == delivery.FollowGrants && g.LeaseHanded
})
if n := said.Load(); n != 0 {
t.Errorf("%d memberships were said on their subject", n)
}
for _, bucket := range []string{grants.Module + "_lease", grants.Module + "_takeover"} {
if _, err := js.KeyValue(ctx, bucket); !errors.Is(err, jetstream.ErrBucketNotFound) {
t.Errorf("the bucket %s: %v, want none", bucket, err)
}
}
}
modules/mesh-grants/cmd/mesh-grants/lease.go
-79
View File
@@ -1,79 +0,0 @@
package main
import (
"context"
"encoding/json"
"fmt"
"sync"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/buscompose"
)
// leaseHand is the process's hand on the controller's lease (novox/hq ADR 0582 clause 3): its key holder, which the
// lease write's cut grants the part alone, and the bucket's expiry, read from the bucket's own stream.
type leaseHand struct {
ctx context.Context
js jetstream.JetStream
mu sync.Mutex
kv jetstream.KeyValue
}
func (l *leaseHand) bucket() (jetstream.KeyValue, error) {
l.mu.Lock()
defer l.mu.Unlock()
if l.kv != nil {
return l.kv, nil
}
ctx, cancel := context.WithTimeout(l.ctx, requestWithin)
defer cancel()
kv, err := l.js.KeyValue(ctx, buscompose.LeaseBucket)
if err != nil {
return nil, fmt.Errorf("the lease's bucket cannot be opened: %w", err)
}
l.kv = kv
return kv, nil
}
func (l *leaseHand) PutHolder(value []byte) error {
kv, err := l.bucket()
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(l.ctx, requestWithin)
defer cancel()
_, err = kv.Put(ctx, buscompose.ControllerLeaseKey, value)
return err
}
func (l *leaseHand) DeleteHolder() error {
kv, err := l.bucket()
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(l.ctx, requestWithin)
defer cancel()
return kv.Delete(ctx, buscompose.ControllerLeaseKey)
}
func (l *leaseHand) Expiry() (time.Duration, error) {
kv, err := l.bucket()
if err != nil {
return 0, err
}
ctx, cancel := context.WithTimeout(l.ctx, requestWithin)
defer cancel()
st, err := kv.Status(ctx)
if err != nil {
return 0, err
}
return st.TTL(), nil
}
// PutTTL writes a key of the part's own state with its own time to live (the state lease).
func (s ownStore) PutTTL(state, key string, value []byte, ttl time.Duration) (uint64, error) {
return s.c.State(state).RenewTTL(key, json.RawMessage(value), ttl)
}
modules/mesh-grants/cmd/mesh-grants/main.go
+10 -26
View File
@@ -4,21 +4,17 @@
// list and each manifest's buckets, logs and streams with the bus composer the controller uses (mesh-sdk buscompose).
//
// **Memberships are published for real** (the operator, 2026-10-25 22:31): since release 6 nobody publishes them, so a
// module placed since cannot serve its seat or reach a new state. Each is published in the part's own state memberships,
// and said on its subject, only when it differs from the one last published there. **A bucket, log or stream a manifest declares that the bus lacks is
// module placed since cannot serve its seat or reach a new state. Each is published in the part's own state memberships
// only when it differs from the one last published there. **A bucket, log or stream a manifest declares that the bus lacks is
// created** (the operator, 2026-10-26 08:54; issue 1790), once per manifest, with the manifest's limits, on the event
// that brought the manifest; one the bus holds is never changed. **The user list is written sealed to the bus node**
// (ADR 0547 clause 7) as an object of the part's own bucket named in `user-list.sealed`, which the bus node follows only
// from take-over. **The rest of the assertions and the grants stay in shadow** until
// the operator's take-over, asked on their channel and run on their yes after its refusals: they are composed, compared with what the bus loaded
// and holds (through the bus module's own read tools) and kept in the part's own state, and take no effect.
// (ADR 0547 clause 7) as an object of the part's own bucket named in `user-list.sealed`, which the bus node follows
// alone since the controller ended by its unassignment (novox/hq ADR 0590, issue 1126). **The grants** the assigner
// reads are written, and **the account consumers** made; a stream the bus holds is never changed, a difference only said.
//
// It holds no timer, and never exits for a refused read, write or publish: a refusal is said and tried again at the next
// event.
//
// **The seat mesh-grants** (novox/hq ADR 0582 clause 1, issue 774): the part holds it for the lease write, serving
// lease-write and lease-hand-back beside its own grants_* tools, and says lease-handed, lease-altered and removal-failed.
//
// Started by the tool runner, with no credential of its own, it serves those tools (a process never serves, ADR 0436).
package main
@@ -38,7 +34,6 @@ import (
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/announce"
"git.novox.be/novox/mesh-sdk/go/asker"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-grants/internal/grants"
@@ -110,21 +105,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
bus := newBusReader(ctx, js)
f.Part = &grants.Part{Read: bus, Call: conn, Store: ownStore{conn}, Emit: emitting{conn},
Publish: memberships{nc: nc}, Create: creating{ctx: ctx, js: js}, Consumers: creating{ctx: ctx, js: js},
Create: creating{ctx: ctx, js: js}, Consumers: creating{ctx: ctx, js: js},
Objects: sealedObjects{ctx: ctx, js: js}, Logf: logger.Printf,
Bus: grants.BusAddress{URL: account.URL, Fingerprint: account.Fingerprint}}
f.Watch = bus
// The operator's word on take-over and hand-back is their yes on their channel: the part asks it itself, and acts on
// the warrant once (mesh-sdk asker), after its refusals.
client := &asker.Client{Module: account.Module, State: conn.State(grants.StateTakeover), Publish: publishVia(ctx, js),
Record: recordVia(ctx, nc, account.Module), Subscribe: decidedVia(nc, account.Module, logger.Printf)}
f.Part.Asker, f.Part.Say = client, sayVia(ctx, js, account.Module)
// The lease write's hand on the lease, and its seat's events (ADR 0582 clauses 3, 4, 6).
f.Part.Lease, f.Part.SaySeat = &leaseHand{ctx: ctx, js: js}, seatSaying(ctx, js)
f.Part.Forget = func(id string) error { return conn.State(grants.StateTakeover).Delete("ask." + id) }
if err := client.Listen(f.Part.Warranted, f.Part.Said); err != nil {
logger.Printf("the operator's answers cannot be heard yet, so take-over and hand-back wait: %v", err)
}
f.Part.Say = sayVia(ctx, js, account.Module)
f.Asked = func(wake func()) error {
return conn.State(grants.StateUserList).Watch(grants.KeyAsked, func(c stdio.StateChange) error {
if !c.Current {
@@ -133,9 +118,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return nil
})
}
// The controller makes this module's durable consumer on EVENTS from the manifest's consumes (filtered on the two
// condition subjects and the vault's account-issued; ADR 0561 §6 gives that to the part from the lease write); it is
// pulled and every message acknowledged, so its pending never grows. Where it is not there yet the pass says so and
// The part makes this module's own durable consumer on EVENTS from the manifest's consumes (filtered on the two
// condition subjects and the vault's account-issued; ADR 0561 §6), as it makes every module's; it is pulled and every message acknowledged, so its pending never grows. Where it is not there yet the pass says so and
// hears again at the next change. The vault's bus-accounts is watched beside it, so a hash is composed either way.
f.Hear = func(handle func(string, []byte)) error {
cons, err := js.Consumer(ctx, "EVENTS", account.Node+"_"+account.Module)
@@ -148,7 +132,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}, jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) }))
return err
}
logger.Printf("started on the account %s: publishing memberships and creating the declared buckets, logs and streams the bus lacks; and writing the user list sealed to the bus node, which the bus node follows from take-over; grants and any change to a held stream in shadow", account.User)
logger.Printf("started on the account %s: publishing memberships, writing grants and the user list sealed to the bus node, which it follows, and creating the account consumers and the declared buckets, logs and streams the bus lacks; a change to a held stream is only said", account.User)
f.Run(ctx)
return nil
}
modules/mesh-grants/cmd/mesh-grants/publish.go
-24
View File
@@ -1,24 +0,0 @@
package main
import (
"fmt"
"github.com/nats-io/nats.go"
"git.novox.be/novox/mesh-sdk/go/membership"
)
// memberships says each membership the part wrote to its state on its subject, `mesh.assignment.<node>.<module>`, as a
// plain message: a notice for a runtime that follows the subject alone, which the assignments stream also keeps while it
// exists. It reads nothing there and asks no stream for an acknowledgement, so the stream's removal at the lease write
// (research 072 row 33) stops no membership (issue 774). The one place the part publishes a membership on the bus.
type memberships struct {
nc *nats.Conn
}
func (m memberships) PublishMembership(node, module string, body []byte) error {
if err := m.nc.Publish(membership.Subject(node, module), body); err != nil {
return fmt.Errorf("the membership of %s on %s: %w", module, node, err)
}
return nil
}
modules/mesh-grants/cmd/mesh-grants/tools.go
+2 -67
View File
@@ -6,7 +6,6 @@ import (
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"mesh-grants/internal/grants"
)
@@ -85,7 +84,7 @@ func toolsOf(p *grants.Part) []stdio.Tool {
Run: func(a map[string]any) (any, error) {
return p.MembershipOf(grants.Answer(a, "node"), grants.Answer(a, "module"))
}},
{Name: grants.ToolUserList, Description: "The bus's user list in shadow (ADR 0547 clauses 6 and 8): the list the grants part " +
{Name: grants.ToolUserList, Description: "The bus's user list (ADR 0547 clauses 6 and 8): the list the grants part " +
"composed (digest, revision, users), what the bus node says it loaded (source, revision, digest, users, error), and how " +
"they differ: users added, users dropped (and of those, gone from placement), each user's grant differences, the " +
"assignments left out, every input the bus does not carry, named; and per user whether the vault holds its hash " +
@@ -98,9 +97,7 @@ func toolsOf(p *grants.Part) []stdio.Tool {
"state (one revision when an input changed, none when not), saying user-list-changed and membership-changed. Answers " +
"the ask and the last pass; grants_user_list shows the pass that answers it. A membership is published unless it would " +
"leave a seat a placed module claims with no holder, or drop a capability its holder still asks for (verified-sender), " +
"which stays a shadow difference, named. The user list, assertions and " +
"grants stay in shadow. It cannot ask the controller to write a module's grant again: the controller serves no such " +
"verb, and the answer says so.",
"which stays a shadow difference, named.",
Run: func(a map[string]any) (any, error) { return p.AskCompose(stdio.CallerOf(a)) }},
{Name: grants.ToolAssert, Description: "One module's buckets, logs and streams (ADR 0547 clause 6), composed from its manifest at " +
"the indexer's trunk with their limits, beside what the bus holds (through the bus module's nats_streams). When the " +
@@ -109,21 +106,6 @@ func toolsOf(p *grants.Part) []stdio.Tool {
"changed, a difference is only said. Answers the comparison and the ask; call again to see what the bus holds.",
Input: schema(map[string]string{"module": "the module"}, "module"),
Run: func(a map[string]any) (any, error) { return p.AssertOne(grants.Answer(a, "module")) }},
{Name: grants.ToolTakeOver, Description: "Ask the operator to switch the bus node to the grants part's list (ADR 0547 clause 8). " +
"The operator's word is their yes on their channel (a warrant), never a caller's name: this tool only asks, once at a time. " +
"After the yes, the switch is recorded and takes effect when the bus node loads the part's list; it is refused while the loaded list could not be read, a user differs between the lists " +
"(an enrolment user or a person the part cannot compose among them), or the vault holds no hash for a user, and the " +
"refusal is said to the operator naming the users. Every act is recorded in the part's state takeover.",
Input: schema(map[string]string{"why": "why, kept with the act"}, "why"),
Run: func(a map[string]any) (any, error) {
return p.TakeOver(stdio.CallerOf(a), grants.Answer(a, "why"))
}},
{Name: grants.ToolHandBack, Description: "Gone: the controller ended by its unassignment (issue 1126), so nothing writes its user list and the grants " +
"part's list is the only one; it says so and asks nothing (ADR 0547 §11).",
Input: schema(map[string]string{"why": "why, kept with the act"}, "why"),
Run: func(a map[string]any) (any, error) {
return p.HandBack(stdio.CallerOf(a), grants.Answer(a, "why"))
}},
{Name: grants.ToolAccounts, Description: "The bus accounts the mesh wants (novox/hq ADR 0561 §2, §9): every account the vault " +
"mints that placement composes (a module's, a process's, a machine's provisioner client's), as the grants part last " +
"wrote it to its state accounts, which the vault alone reads: user, kind, node, module, process, the bus's address and " +
@@ -160,56 +142,9 @@ func toolsOf(p *grants.Part) []stdio.Tool {
}
return p.AcceptHashes(stdio.CallerOf(a), users, grants.Answer(a, "why"))
}},
{Name: grants.ToolTakeOverState, Description: "Where the switch stands: the list the bus node follows, the ask open now, the request, and " +
"every act with what became of it (asked, switched, declined, expired, refused and why).",
Input: schema(nil),
Run: func(map[string]any) (any, error) { return p.TakeOverState() }},
{Name: grants.ToolLeaseCheck, Description: "The controller's lease and the controller's hand-over, in one call from bus state (issue 774, " +
"design 47 L3; novox/hq ADR 0547 §11, ADR 0561 §3, ADR 0571 §13): who holds the controller lease now, since when, and what " +
"the holder and the parts watching the lease do because of it; what still depends on the controller: every module whose " +
"manifest at the indexer's trunk reads a mesh-controller state, invokes a mesh-controller verb or tool, consumes or waits on " +
"its events, with which one and the record that keeps it until the lease write (the named keep list), the unkept ones by module; " +
"the controller's remaining live duties as the bus shows them: each of its streams and buckets someone consumes (with the " +
"durable consumers by name), and its standing assignments; and a dry run of the lease write's check (the take-over check that " +
"hands the lease to a holder that is no controller): whether it would pass now, at the write with only the three kept to the " +
"write and before the write's delivery with the whole list, each with its exact reasons, and the write's other conditions " +
"with the tool that answers each. Asks the controller nothing; writes nothing. Read-only.",
Input: schema(nil),
Run: func(map[string]any) (any, error) { return p.LeaseCheck() }},
// The seat's two verbs (ADR 0582 clauses 1, 5): asked of the operator by the part's process.
{Name: grants.SeatTool(grants.ActLeaseWrite), Description: seatVerb(grants.ActLeaseWrite).Description,
Input: seatVerb(grants.ActLeaseWrite).Input,
Run: func(a map[string]any) (any, error) {
return p.LeaseWrite(stdio.CallerOf(a), grants.Answer(a, "why"))
}},
{Name: grants.SeatTool(grants.ActLeaseHandBack), Description: seatVerb(grants.ActLeaseHandBack).Description,
Input: seatVerb(grants.ActLeaseHandBack).Input,
Run: func(a map[string]any) (any, error) {
return p.LeaseHandBack(stdio.CallerOf(a), grants.Answer(a, "why"))
}},
{Name: grants.ToolRemovalDue, Description: "Write the lease write's removal-due again with a short time to live, so " +
"its expiry marker starts the day-later removal of the controller's consumer of DEAD_LETTER_NOTICES and the " +
"stream ASSIGNMENTS (novox/hq ADR 0582 clause 6): the recovery for a marker missed while the part was down, the " +
"removal still started by the marker and nothing else. Refused unless the lease is handed to mesh-grants and the " +
"removal has not run. Recorded as a hand act.",
Input: schema(map[string]string{"why": "why, kept with the hand act", "ttl": "the time to live, e.g. 10s (1s to 24h)"}, "why", "ttl"),
Run: func(a map[string]any) (any, error) {
return p.RemovalDue(stdio.CallerOf(a), grants.Answer(a, "why"), grants.Answer(a, "ttl"))
}},
}
}
// seatVerb is the seat mesh-grants' verb as mesh-sdk declares it.
func seatVerb(name string) catalogue.Verb {
seat, _ := catalogue.SeatNamed(catalogue.GrantsSeat)
for _, v := range seat.Serves {
if v.Name == name {
return v
}
}
return catalogue.Verb{Name: name}
}
// serveTools serves the tools to the runtime until it stops.
func serveTools() error {
p := &grants.Part{Read: runtimeReader{}, Call: runtimeBus{}, Store: runtimeStore{}, Emit: runtimeBus{}}
modules/mesh-grants/grants_test.go
+156 -469
View File
@@ -19,14 +19,12 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/asker"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-sdk/go/membership"
"git.novox.be/novox/mesh-sdk/go/outward"
"git.novox.be/novox/mesh-sdk/go/seal"
"mesh-grants/internal/grants"
@@ -398,7 +396,7 @@ func fixture() *fakeBus {
}
func partOn(b *fakeBus) *grants.Part {
return &grants.Part{Read: b, Call: b, Store: ownStore{b}, Emit: b, Publish: b, Now: func() time.Time { return time.Date(2026, 10, 25, 22, 0, 0, 0, time.UTC) }}
return &grants.Part{Read: b, Call: b, Store: ownStore{b}, Emit: b, Now: func() time.Time { return time.Date(2026, 10, 25, 22, 0, 0, 0, time.UTC) }}
}
// shadowOf is the list the part composes from b's records now.
@@ -437,18 +435,10 @@ func TestGrants(t *testing.T) {
if err != nil {
t.Fatalf("mesh-sdk cannot decode the manifest: %v", err)
}
// The part holds the seat mesh-grants for the lease write (ADR 0582 clause 1, issue 774, 774/S14), as mesh-sdk's seat
// list declares it.
if seat, ok := catalogue.SeatNamed(catalogue.GrantsSeat); m.Module != "mesh-grants" || len(m.Claims) != 1 || !ok ||
catalogue.CanHold(m, seat) != nil {
t.Errorf("module %q claims %v: the part holds mesh-grants alone (%v)", m.Module, m.Claims, catalogue.CanHold(m, seat))
}
var keyTTL bool
for _, s := range m.State {
keyTTL = keyTTL || (s.Name == grants.StateLease && s.KeyTTL)
}
if !keyTTL {
t.Errorf("the state %s gives no time to live per key, which removal-due needs", grants.StateLease)
// The part claims no seat: the seat mesh-grants served the lease write alone, which went with the controller
// (novox/hq ADR 0590 §3, issue 1126).
if m.Module != "mesh-grants" || len(m.Claims) != 0 {
t.Errorf("module %q claims %v: the part claims no seat", m.Module, m.Claims)
}
for _, row := range []struct {
what string
@@ -476,15 +466,14 @@ func TestGrants(t *testing.T) {
}
})
t.Run("no timer, no update or delete, a publish only of memberships, a create only of declared streams", func(t *testing.T) {
t.Run("no timer, no update or delete, a publish only of conditions, a create only of declared streams", func(t *testing.T) {
forbidden := []string{"time.Tick", "time.NewTicker", "time.Sleep", "time.After", "time.NewTimer", "time.AfterFunc",
"PublishMsg", "CreateKeyValue", "CreateOrUpdate", "UpdateStream", "UpdateKeyValue", "DeleteStream", "PurgeStream",
"DeleteKeyValue", "CreateObjectStore", "EnsureStream", "RaiseBuckets", "RaiseLogs", ".Create("}
// The one publish: a membership, on its own subject, in publish.go; the one create: a declared stream the bus lacks,
// The one publish: a condition, on the seat operator-channel's subject, in asking.go; a membership is written to
// the part's state alone, never said on its subject (issue 1126). The one create: a declared stream the bus lacks,
// in create.go (issue 1790).
publishOnly := filepath.Join("cmd", "mesh-grants", "publish.go")
createOnly := filepath.Join("cmd", "mesh-grants", "create.go")
// And the operator's asks and conditions, on the seat operator-channel's subjects, in asking.go (ADR 0547 clause 8).
asksOnly := filepath.Join("cmd", "mesh-grants", "asking.go")
sealedOnly := filepath.Join("cmd", "mesh-grants", "sealed.go")
files := 0
@@ -495,15 +484,11 @@ func TestGrants(t *testing.T) {
}
files++
src, err := os.ReadFile(path)
if path != publishOnly && path != asksOnly && (strings.Contains(string(src), ".Publish(") || strings.Contains(string(src), "mesh.assignment")) {
t.Errorf("%s publishes: only publish.go publishes memberships, and asking.go the operator's asks", path)
if path != asksOnly && (strings.Contains(string(src), ".Publish(") || strings.Contains(string(src), "mesh.assignment")) {
t.Errorf("%s publishes: only asking.go publishes, a condition", path)
}
// And the seat mesh-grants' events (ADR 0582 clauses 3, 4, 6; issue 774).
if path == asksOnly && (strings.Count(string(src), ".Publish(") != 3 || strings.Contains(string(src), "mesh.assignment")) {
t.Errorf("asking.go publishes other than an ask, a cancel and a condition, and the seat's events, through js.Publish thrice")
}
if path == publishOnly && strings.Count(string(src), ".Publish(") != 1 {
t.Errorf("publish.go publishes other than one membership")
if path == asksOnly && (strings.Count(string(src), ".Publish(") != 1 || strings.Contains(string(src), "mesh.assignment")) {
t.Errorf("asking.go publishes other than a condition, through js.Publish once")
}
if n := strings.Count(string(src), "CreateStream"); (path == createOnly && n != 1) || (path != createOnly && n != 0) {
t.Errorf("%s holds CreateStream %d times: only create.go creates, once", path, n)
@@ -926,10 +911,6 @@ func TestGrants(t *testing.T) {
!slices.Contains(cr.LeftOut, "b.mod-y") {
t.Errorf("a hash heard = %+v", after.UserList.Credentials)
}
why := strings.Join(p.Decide(), "; ")
if !strings.Contains(why, "no hash for") || strings.Contains(why, "a.mod-x,") {
t.Errorf("take-over is refused for %q, want the users with no hash named", why)
}
for state, keys := range b.own {
for k, v := range keys {
if strings.Contains(string(v), "secrethashbytes") {
@@ -1042,55 +1023,14 @@ func TestGrants(t *testing.T) {
if strings.Contains(row.name, "login lost") {
cr.LosesLogin = []string{"novox.telegram"}
}
// Removing, and the users with no hash in the vault, are what the yes is refused for.
// Removing, and the users with no hash in the vault, are what a list in force is held for.
var removing []string
for _, r := range c.Removing() {
removing = append(removing, strings.Fields(r)[0])
}
removing = append(removing, cr.LosesLogin...)
if !slices.Equal(removing, row.removing) || len(c.AddedRights()) != row.adds {
t.Errorf("take-over refused for %v, want %v; added rights of %d users, want %d", c.Removing(), row.removing, len(c.AddedRights()), row.adds)
}
// The ask's words pass the operator channel's content rule, read at a glance and fit one message; where only
// rights are added they name every user whose rights are added, and where a loss stands they never say none is
// taken away and name the users the yes would be refused for.
summary, points, list := grants.TakeOverWords(c, cr)
if row.removing != nil {
if strings.Contains(summary, "takes none away") {
t.Errorf("the ask says none is taken away with a loss: %q", summary)
}
for _, u := range row.removing {
if !strings.Contains(list, u) {
t.Errorf("the ask does not name %s, refused for: %s", u, list)
}
}
}
if row.live != nil && row.removing == nil && c.Equal() {
t.Error("the captured comparison is equal: the old guard would not have refused it")
}
if row.adds > 0 {
if r, ok := outward.Check(summary + "\n" + strings.Join(points, "\n") + "\n" + list); !ok {
t.Errorf("the ask's words break the content rule: %s", r)
}
}
if row.adds > 0 && row.removing == nil {
expl := summary + "\n" + strings.Join(points, "\n")
for _, text := range []string{expl, list} {
if r, ok := outward.Check(text); !ok {
t.Errorf("the ask's words break the content rule: %s", r)
}
}
if err := asks.Glance(expl); err != nil || len([]rune(list)) > grants.AddedWordsBudget {
t.Errorf("the ask's words do not fit: %v, a list of %d characters", err, len([]rune(list)))
}
for _, d := range c.AddedRights() {
if !strings.Contains(list, d.User) && !strings.Contains(list, "more users") {
t.Errorf("the ask does not name %s", d.User)
}
}
}
if row.live != nil && row.removing == nil && (!strings.Contains(summary, "adds 391 rights to 41 users") || !strings.Contains(list, "node.ace, node.novox: 36 reads, 1 call, 1 event each")) {
t.Errorf("the captured ask says %q and lists %q", summary, list)
t.Errorf("removing %v, want %v; added rights of %d users, want %d", c.Removing(), row.removing, len(c.AddedRights()), row.adds)
}
if row.name == "grants differ" && (len(c.Differ[0].PublishAdded) != 1 || c.Differ[0].PublishDropped[0] != "extra.subject") {
t.Errorf("the grant difference is not named: %+v", c.Differ[0])
@@ -1313,11 +1253,12 @@ func TestGrants(t *testing.T) {
if !first.Written || first.Revision != 1 || !first.UserList.Compare.Equal() || first.UserList.Users != 6 {
t.Fatalf("first composition = %+v", first)
}
// Written in shadow, the bus node told to follow the controller's list; every user left out while the vault holds
// no hash; a tool's part (no object bucket) writes none.
if g, _, had := p.Sealed(); !had || g.Follow != delivery.FollowController || !g.LeaseHanded || g.Revision != 1 || g.Node != "a" ||
g.Users != 0 || len(g.LeftOut) != 6 || len(objs.objects) != 1 || !strings.Contains(open(g), "accounts {") {
t.Errorf("the sealed list in shadow = %+v, %d objects", g, len(objs.objects))
// The bus node follows the part's list alone (issue 1126): a first list that would leave out every loaded user
// while the vault holds no hash is held, nothing sealed, the key saying follow grants; a tool's part (no object
// bucket) writes none.
if g, _, had := p.Sealed(); !had || g.Follow != delivery.FollowGrants || !g.LeaseHanded || g.Digest != "" ||
len(objs.objects) != 0 || first.Sealed == nil || !strings.Contains(first.Sealed.Why, "would be left out") {
t.Errorf("the first sealed list = %+v, %d objects, %+v", g, len(objs.objects), first.Sealed)
}
if _, _, had := partOn(fixture()).Sealed(); had {
t.Error("a part read before any pass holds a sealed list")
@@ -1378,39 +1319,8 @@ func TestGrants(t *testing.T) {
t.Errorf("grants = %v", gr)
}
b.loaded = append(b.loaded, grants.User{User: "b.gone"})
fa := &fakeAsker{wheres: map[string]*asks.Warrant{}}
var forgot []string
n := 0
p.Nonce, p.Forget = func() string { n++; return fmt.Sprintf("%04d", n) }, func(id string) error { forgot = append(forgot, id); return nil }
var said []asks.Condition
p.Asker, p.Say = fa, func(c asks.Condition) error { said = append(said, c); return nil }
warrant := func(id string) asks.Warrant { return asks.Warrant{Ask: id, Outcome: asks.OutcomeChosen} }
last := func() grants.Act {
raw, _, _, _ := b.ownGet(grants.StateTakeover, grants.KeySwitch)
var s grants.Switch
_ = json.Unmarshal(raw, &s)
if s.Open != "" && s.Acts[len(s.Acts)-1].Said != "asked" {
t.Errorf("an ask is open and its act is not asked: %+v", s)
}
return s.Acts[len(s.Acts)-1]
}
asked := func(act string) string {
t.Helper()
var res struct{ Asked string }
var err error
var out any
if act == "take-over" {
out, err = p.TakeOver("shanks.node-tools", "a row")
}
if err != nil {
t.Fatalf("%s: %v", act, err)
}
raw, _ := json.Marshal(out)
_ = json.Unmarshal(raw, &res)
return res.Asked
}
yes, no := asks.Option{ID: "yes"}, asks.Option{ID: "no"}
p.Say = func(c asks.Condition) error { said = append(said, c); return nil }
// settled is the placement of the lists agreeing, every hash held, and the loaded list as edit leaves it.
settled := func(edit func(modX *grants.User)) func() {
return func() {
@@ -1428,320 +1338,145 @@ func TestGrants(t *testing.T) {
}
}
lacks := func(modX *grants.User) { modX.Publish = modX.Publish[1:] } // the shadow adds a right (ADR 0562)
for _, row := range []struct {
name, act string
option asks.Option
setup func()
said, never []string // the refusal's words and the users it must not name
state string
outcome string
// between changes the lists after the ask and before the yes; asked are words the ask must carry.
between func()
asked []string
}{
{"a yes while a user differs is refused and said, naming it", "take-over", yes, func() {}, []string{"users differ", "b.mod-x"}, []string{"b.gone"}, grants.Shadow, "refused", nil, nil},
{"a no leaves the switch", "take-over", no, func() {}, nil, nil, grants.Shadow, "declined", nil, nil},
{"a yes while a loaded user would lose a right is refused, naming it", "take-over", yes, settled(func(modX *grants.User) {
modX.Publish = append(modX.Publish, "extra.subject")
}), []string{"other than by an added right", "a.mod-x (would lose 1 publish)"}, []string{"b.gone"}, grants.Shadow, "refused", nil, nil},
{"a yes after the added rights changed since the ask is refused", "take-over", yes, settled(lacks), []string{"changed since you were asked"}, nil, grants.Shadow, "refused",
func() {
lacks(&b.loaded[slices.IndexFunc(b.loaded, func(u grants.User) bool { return u.User == "a.mod-x" })])
}, []string{"a.mod-x: 1 read"}},
{"a yes with only added rights and every hash held records the switch, the ask naming them", "take-over", yes, settled(lacks), nil, nil, grants.TookOver, "recorded",
nil, []string{"adds 1 right to 1 user and takes none away", "a.mod-x: 1 read"}},
} {
t.Run(row.name, func(t *testing.T) {
row.setup()
before := len(fa.asks)
id := asked(row.act)
if _, err := p.TakeOver("shanks.node-tools", "again"); err == nil || !strings.Contains(err.Error(), "already asked") {
t.Errorf("a second ask while one is open = %v", err)
}
p.Compose()
p.Compose()
if len(fa.asks) != before+1 || fa.asks[before].ID != id {
t.Fatalf("the operator was asked %d times, want once: %+v", len(fa.asks)-before, fa.asks)
}
ask := fa.asks[before]
if err := ask.Check(time.Date(2026, 10, 25, 22, 0, 0, 0, time.UTC)); err != nil {
t.Errorf("the router would refuse the ask: %v", err)
}
if yesOpt, ok := ask.Option("yes"); !ok || yesOpt.Level != asks.Approve || !strings.HasPrefix(yesOpt.Binds, "sha256:") {
t.Errorf("the yes does not approve and bind its act: %+v", yesOpt)
}
for _, text := range []string{ask.Explanation, ask.Details} {
if r, ok := outward.Check(text); !ok {
t.Errorf("the ask breaks the content rule: %s", r)
}
}
if err := asks.Glance(ask.Explanation); err != nil {
t.Error(err)
}
for _, w := range row.asked {
if !strings.Contains(ask.Explanation+"\n"+ask.Details, w) {
t.Errorf("the ask does not say %q: %s\n%s", w, ask.Explanation, ask.Details)
}
}
if row.between != nil {
row.between()
}
said = nil
calls := len(b.calls)
if err := p.Warranted(row.option, warrant(id)); err != nil {
t.Fatal(err)
}
// 1953/D8 (ADR 0565 §11): a switch made is recorded in the hand-act log's seat; one refused or declined is not.
if recorded := slices.Contains(b.calls[calls:], grants.CallRecordWarrant); recorded != (row.outcome == "recorded") {
t.Errorf("recorded in the hand-act log %v, the act %s", recorded, row.outcome)
}
l := last()
if l.Said != row.outcome || l.Ask != id || p.SwitchState() != row.state {
t.Errorf("the act = %+v, the switch %s; want %s and %s", l, p.SwitchState(), row.outcome, row.state)
}
for _, w := range row.said {
if !strings.Contains(l.Refused, w) || (row.outcome == "refused" && (len(said) != 1 || !strings.Contains(said[0].Explanation, w))) {
t.Errorf("the refusal %q (said %v) does not say %q", l.Refused, said, w)
}
}
for _, w := range row.never {
if strings.Contains(l.Refused, w) {
t.Errorf("the refusal names %q", w)
}
}
// A recorded switch says no refusal and clears the one a refused yes left raised.
if row.outcome == "recorded" && row.act == "take-over" && (len(said) != 1 || said[0].State != asks.StateCleared || said[0].Key != "mesh-grants.take-over-refused") {
t.Errorf("a recorded switch did not clear the refusal: %v", said)
}
if row.outcome == "recorded" && !strings.Contains(fmt.Sprint(p.TakeOverState()), "bus_node_loaded_grants_list:not yet") {
t.Errorf("the state claims the bus node follows with no loaded list: %v", fmt.Sprint(p.TakeOverState()))
}
// The key the bus node follows turns with the switch at once, and the state says switched only once the bus
// node loaded the grants part's written list.
g, _, _ := p.Sealed()
if want := map[string]string{grants.TookOver: delivery.FollowGrants, grants.Shadow: delivery.FollowController}[p.SwitchState()]; g.Follow != want || !g.LeaseHanded {
t.Errorf("the key says follow %s, lease handed %v, with the switch %s and no lease key (1126/A1/AC2)", g.Follow, g.LeaseHanded, p.SwitchState())
}
effect := func() string { st, _ := p.TakeOverState(); return fmt.Sprint(st.(map[string]any)["effect"]) }
if row.outcome == "recorded" && row.act == "take-over" {
if e := effect(); strings.HasPrefix(e, "switched") || !strings.Contains(e, "not in effect") {
t.Errorf("effect before the bus node loaded = %s", e)
}
if users := strings.Count(open(g), "{ user: "); users != g.Users || g.Users == 0 || len(g.LeftOut) != 0 {
t.Errorf("the sealed list holds %d users, the key says %d, left out %v", users, g.Users, g.LeftOut)
}
withKey(delivery.UserListWritten{Source: delivery.GrantsUserListSource, Revision: g.Revision, Wanted: g.Digest, Digest: g.Digest, Users: g.Users, Follows: delivery.FollowGrants})
if e := effect(); !strings.HasPrefix(e, "switched") {
t.Errorf("effect once the bus node loaded the written list = %s", e)
}
// Steady (issue 1126, 2026-10-28 00:16: the shadow list moved every 3 s, 99270 to 99357, with nothing
// differing): the bus node saying again that it loaded the same list, at a later time and the key's
// later revision, moves neither the shadow list nor the sealed key.
_, sealedRev, _ := p.Sealed()
p.Compose()
for i := 0; i < 3; i++ {
withKey(delivery.UserListWritten{Source: delivery.GrantsUserListSource, Revision: sealedRev + uint64(i+1), Wanted: g.Digest,
Digest: g.Digest, Users: g.Users, Follows: delivery.FollowGrants, At: time.Date(2026, 10, 28, 0, 16, i, 0, time.UTC)})
if res := p.Compose(); res.Written {
t.Errorf("pass %d: the shadow list is written again for the bus node saying the same list again (revision %d)", i, res.Revision)
}
if _, rev, _ := p.Sealed(); rev != sealedRev {
t.Errorf("pass %d: the sealed key is written again with nothing changed: revision %d, was %d", i, rev, sealedRev)
}
}
withKey(loadedBefore)
// After take-over a list composed past a failed read, or one dropping a user of the list in force for
// want of a hash, is not sealed: the list in force stays, and the pass says why.
b.refused[grants.ReadAccounts] = true
if res := p.Compose(); res.Sealed == nil || !strings.Contains(res.Sealed.Why, "a read of this pass failed") {
t.Errorf("a pass past a failed read = %+v", res.Sealed)
}
// While the hold lasts the warning stands, said once however many passes find it; it names why.
p.Compose()
nHeld := 0
for _, c := range said {
if c.Key == grants.HeldCondition && c.State == asks.StateRaised && c.Severity == asks.SeverityWarning && strings.Contains(c.Summary, "a read of this pass failed") {
nHeld++
}
}
if nHeld != 1 {
t.Errorf("the held list raised the warning %d times, want once: %+v", nHeld, said)
}
b.refused[grants.ReadAccounts] = false
if now, _, _ := p.Sealed(); now.Digest != g.Digest || now.Object != g.Object {
t.Errorf("the list in force moved past a failed read: %s, was %s", now.Digest, g.Digest)
}
b.mu.Lock()
hash := b.states[grants.ReadAccounts]["a.mod-x"]
delete(b.states[grants.ReadAccounts], "a.mod-x")
b.mu.Unlock()
if res := p.Compose(); res.Sealed == nil || !strings.Contains(res.Sealed.Why, "a.mod-x would be left out") {
t.Errorf("a pass that drops a.mod-x's hash = %+v", res.Sealed)
}
if now, _, _ := p.Sealed(); now.Digest != g.Digest {
t.Errorf("the list in force moved without a.mod-x: %s", now.Digest)
}
// A list in force written before the part kept its users (no sealed-users): checked against the users the
// bus loaded, so the first list after take-over has the same protection.
b.mu.Lock()
delete(b.own[grants.StateUserList], grants.KeyWrittenUsers)
b.mu.Unlock()
if res := p.Compose(); res.Sealed == nil || !strings.Contains(res.Sealed.Why, "a.mod-x would be left out") {
t.Errorf("a pass with no kept users that drops a.mod-x's hash = %+v", res.Sealed)
}
if now, _, _ := p.Sealed(); now.Digest != g.Digest {
t.Errorf("the list in force moved without a.mod-x and no kept users: %s", now.Digest)
}
b.put(grants.ReadAccounts, "a.mod-x", json.RawMessage(hash))
if p.Compose(); func() bool { now, _, _ := p.Sealed(); return now.Digest != g.Digest }() {
t.Errorf("the list in force moved with every read and hash back")
}
if last := said[len(said)-1]; last.Key != grants.HeldCondition || last.State != asks.StateCleared {
t.Errorf("the held warning is not cleared once nothing is held: %+v", last)
}
// A part started while the warning stood (the said state is in memory only) clears it on its first pass.
var fresh []asks.Condition
fp := partOn(b)
fp.Objects = p.Objects
fp.Asker, fp.Say = fa, func(c asks.Condition) error {
if c.Key != grants.GrantMissingCondition && c.Key != grants.RefusedCondition { // their own rows
fresh = append(fresh, c)
}
return nil
}
fp.Compose()
if len(fresh) == 0 || fresh[0].Key != grants.HeldCondition || fresh[0].State != asks.StateCleared {
t.Errorf("a fresh part's first pass does not say the held warning is cleared: %+v", fresh)
}
// Issue 1945, 2026-10-27 09:21: a list that would leave a seat the list in force holds with no holder,
// while a placed module still claims it, is held, as such a membership is: not written, the warning
// raised naming the seat. A list that only adds is written; one whose loaded grants cannot be read is held.
inForce := slices.Clone(b.loaded)
seat, verb := brokerSeat()
seatGrant := "mesh.seat." + seat + ".call." + verb + ".*"
for i := range b.loaded {
if b.loaded[i].User == "a.nats" {
b.loaded[i].Subscribe = append(slices.Clone(b.loaded[i].Subscribe), seatGrant)
}
}
b.assign("a", "mod-y")
b.put(grants.ReadAccounts, "a.mod-y", map[string]any{"user": "a.mod-y", "hash": "$2a$10$x", "generation": 1})
said = nil
res := p.Compose()
if res.Sealed == nil || !strings.Contains(res.Sealed.Why, "publishing would lose the last holder of "+seat) {
t.Errorf("a list losing a seat grant = %+v", res.Sealed)
}
if now, _, _ := p.Sealed(); now.Digest != g.Digest {
t.Errorf("a list losing a seat grant was written: %s, was %s", now.Digest, g.Digest)
}
if len(said) != 1 || said[0].Key != grants.HeldCondition || said[0].State != asks.StateRaised || !strings.Contains(said[0].Summary, "the last holder of "+seat) {
t.Errorf("the held list did not raise the warning naming the lost grant: %+v", said)
}
b.loaded = inForce
if p.Compose(); func() bool { now, _, _ := p.Sealed(); return now.Digest == g.Digest }() {
t.Errorf("a list that only adds a.mod-y was not written")
}
written, _, _ := p.Sealed()
b.mu.Lock()
b.refuseUsers = true
b.mu.Unlock()
b.mu.Lock()
b.unassign("a", "mod-y")
b.mu.Unlock()
if res := p.Compose(); res.Sealed == nil || !strings.Contains(res.Sealed.Why, "the grants of the list in force cannot be read") {
t.Errorf("a pass whose loaded list cannot be read = %+v", res.Sealed)
}
if now, _, _ := p.Sealed(); now.Digest != written.Digest {
t.Errorf("a list was written while the loaded list could not be read")
}
b.mu.Lock()
b.refuseUsers = false
b.mu.Unlock()
}
})
// The bus node follows the part's list alone since the controller's end (novox/hq ADR 0590, issue 1126): with the
// lists apart only by an added right and every hash held, the list is sealed whole, the key saying follow grants.
settled(lacks)()
p.Compose()
g, _, _ := p.Sealed()
if g.Follow != delivery.FollowGrants || !g.LeaseHanded {
t.Errorf("the key says follow %s, lease handed %v; want grants with no lease key (1126/A1/AC2)", g.Follow, g.LeaseHanded)
}
// 1126/A1: the controller ended by its unassignment, so nothing writes its list and a hand-back is gone.
t.Run("hand-back is gone at once, with nothing asked", func(t *testing.T) {
n := len(fa.asks)
if _, err := p.HandBack("shanks.node-tools", "a row"); err == nil || !strings.HasPrefix(err.Error(), "gone") {
t.Errorf("hand-back = %v", err)
if users := strings.Count(open(g), "{ user: "); users != g.Users || g.Users == 0 || len(g.LeftOut) != 0 {
t.Errorf("the sealed list holds %d users, the key says %d, left out %v", users, g.Users, g.LeftOut)
}
withKey(delivery.UserListWritten{Source: delivery.GrantsUserListSource, Revision: g.Revision, Wanted: g.Digest, Digest: g.Digest, Users: g.Users, Follows: delivery.FollowGrants})
// Steady (issue 1126, 2026-10-28 00:16: the shadow list moved every 3 s, 99270 to 99357, with nothing
// differing): the bus node saying again that it loaded the same list, at a later time and the key's
// later revision, moves neither the shadow list nor the sealed key.
_, sealedRev, _ := p.Sealed()
p.Compose()
for i := 0; i < 3; i++ {
withKey(delivery.UserListWritten{Source: delivery.GrantsUserListSource, Revision: sealedRev + uint64(i+1), Wanted: g.Digest,
Digest: g.Digest, Users: g.Users, Follows: delivery.FollowGrants, At: time.Date(2026, 10, 28, 0, 16, i, 0, time.UTC)})
if res := p.Compose(); res.Written {
t.Errorf("pass %d: the shadow list is written again for the bus node saying the same list again (revision %d)", i, res.Revision)
}
p.Compose()
if len(fa.asks) != n {
t.Errorf("a refused hand-back asked the operator")
if _, rev, _ := p.Sealed(); rev != sealedRev {
t.Errorf("pass %d: the sealed key is written again with nothing changed: revision %d, was %d", i, rev, sealedRev)
}
})
t.Run("an ask that can stay open is closed every way and can be asked again", func(t *testing.T) {
// The rows above end taken over and a hand-back is gone (1126/A1): asked again from a part in shadow.
delete(b.own[grants.StateTakeover], grants.KeySwitch)
// A warrant the process missed is caught up from the router's record, whatever its outcome.
id := asked("take-over")
p.Compose()
fa.wheres[id] = &asks.Warrant{Ask: id, Outcome: asks.OutcomeChosen, Option: "no"}
p.Compose()
if l := last(); l.Ask != id || l.Said != "declined" {
t.Errorf("a missed warrant was not caught up: %+v", l)
}
withKey(loadedBefore)
// After take-over a list composed past a failed read, or one dropping a user of the list in force for
// want of a hash, is not sealed: the list in force stays, and the pass says why.
b.refused[grants.ReadAccounts] = true
if res := p.Compose(); res.Sealed == nil || !strings.Contains(res.Sealed.Why, "a read of this pass failed") {
t.Errorf("a pass past a failed read = %+v", res.Sealed)
}
// While the hold lasts the warning stands, said once however many passes find it; it names why.
p.Compose()
nHeld := 0
for _, c := range said {
if c.Key == grants.HeldCondition && c.State == asks.StateRaised && c.Severity == asks.SeverityWarning && strings.Contains(c.Summary, "a read of this pass failed") {
nHeld++
}
// The router ended it with no choice, and the process was not told.
id = asked("take-over")
p.Compose()
fa.wheres[id] = &asks.Warrant{Ask: id, Outcome: asks.OutcomeCancelled}
p.Compose()
if l := last(); l.Ask != id || !strings.Contains(l.Said, "cancelled") {
t.Errorf("an ask the router ended was not closed: %+v", l)
}
if nHeld != 1 {
t.Errorf("the held list raised the warning %d times, want once: %+v", nHeld, said)
}
b.refused[grants.ReadAccounts] = false
if now, _, _ := p.Sealed(); now.Digest != g.Digest || now.Object != g.Object {
t.Errorf("the list in force moved past a failed read: %s, was %s", now.Digest, g.Digest)
}
b.mu.Lock()
hash := b.states[grants.ReadAccounts]["a.mod-x"]
delete(b.states[grants.ReadAccounts], "a.mod-x")
b.mu.Unlock()
if res := p.Compose(); res.Sealed == nil || !strings.Contains(res.Sealed.Why, "a.mod-x would be left out") {
t.Errorf("a pass that drops a.mod-x's hash = %+v", res.Sealed)
}
if now, _, _ := p.Sealed(); now.Digest != g.Digest {
t.Errorf("the list in force moved without a.mod-x: %s", now.Digest)
}
// A list in force written before the part kept its users (no sealed-users): checked against the users the
// bus loaded, so the first list after take-over has the same protection.
b.mu.Lock()
delete(b.own[grants.StateUserList], grants.KeyWrittenUsers)
b.mu.Unlock()
if res := p.Compose(); res.Sealed == nil || !strings.Contains(res.Sealed.Why, "a.mod-x would be left out") {
t.Errorf("a pass with no kept users that drops a.mod-x's hash = %+v", res.Sealed)
}
if now, _, _ := p.Sealed(); now.Digest != g.Digest {
t.Errorf("the list in force moved without a.mod-x and no kept users: %s", now.Digest)
}
b.put(grants.ReadAccounts, "a.mod-x", json.RawMessage(hash))
if p.Compose(); func() bool { now, _, _ := p.Sealed(); return now.Digest != g.Digest }() {
t.Errorf("the list in force moved with every read and hash back")
}
if last := said[len(said)-1]; last.Key != grants.HeldCondition || last.State != asks.StateCleared {
t.Errorf("the held warning is not cleared once nothing is held: %+v", last)
}
// A part started while the warning stood (the said state is in memory only) clears it on its first pass.
var fresh []asks.Condition
fp := partOn(b)
fp.Objects = p.Objects
fp.Say = func(c asks.Condition) error {
if c.Key != grants.GrantMissingCondition && c.Key != grants.RefusedCondition { // their own rows
fresh = append(fresh, c)
}
// Said closes on any outcome for the open id, and none for another id.
id = asked("take-over")
p.Said(asker.Refused, asks.Warrant{Ask: "other", Outcome: asks.OutcomeRefused}, nil)
if _, err := p.TakeOver("x", "y"); err == nil {
t.Errorf("a warrant for another ask closed the open one")
return nil
}
fp.Compose()
if len(fresh) == 0 || fresh[0].Key != grants.HeldCondition || fresh[0].State != asks.StateCleared {
t.Errorf("a fresh part's first pass does not say the held warning is cleared: %+v", fresh)
}
// Issue 1945, 2026-10-27 09:21: a list that would leave a seat the list in force holds with no holder,
// while a placed module still claims it, is held, as such a membership is: not written, the warning
// raised naming the seat. A list that only adds is written; one whose loaded grants cannot be read is held.
inForce := slices.Clone(b.loaded)
seat, verb := brokerSeat()
seatGrant := "mesh.seat." + seat + ".call." + verb + ".*"
for i := range b.loaded {
if b.loaded[i].User == "a.nats" {
b.loaded[i].Subscribe = append(slices.Clone(b.loaded[i].Subscribe), seatGrant)
}
p.Said(asker.Refused, asks.Warrant{Ask: id, Outcome: asks.OutcomeRefused}, nil)
// A warrant for an ask that is not the open one switches nothing.
if err := p.Warranted(yes, warrant("take-over-other")); err == nil {
t.Errorf("a warrant for an ask that is not open was acted on")
}
// Past its expiry and a margin the part closes it itself; the clock moves, not a timer.
id = asked("take-over")
now := p.Now
p.Now = func() time.Time { return now().Add(25 * time.Hour) }
p.Compose()
p.Now = now
if l := last(); l.Ask != id || l.Said != string(asks.OutcomeExpired) {
t.Errorf("an expired ask was not closed: %+v", l)
}
// An ask that could not be published is closed as refused, its half-made record taken away, and is not retried.
fa.fail = true
id = asked("take-over")
p.Compose()
p.Compose()
if l := last(); l.Ask != id || l.Said != "refused" || !slices.Contains(forgot, id) || !slices.Contains(fa.cancels, id) {
t.Errorf("a failed ask stays open, or is not taken back at the router: %+v, forgot %v, cancelled %v", l, forgot, fa.cancels)
}
fa.fail = false
// An answer the part's own record cannot take yet (its state out of reach) leaves the ask open, closed by
// nothing, and is taken at a later pass.
id = asked("take-over")
p.Compose()
fa.wheres[id] = &asks.Warrant{Ask: id, Outcome: asks.OutcomeChosen, Option: "no"}
fa.takeErr = errors.New("nats: no responders available for request")
p.Compose()
if l := last(); l.Ask != id || l.Said != "asked" {
t.Errorf("an answer not taken closed the ask: %+v", l)
}
fa.takeErr = nil
p.Compose()
if l := last(); l.Ask != id || l.Said != "declined" {
t.Errorf("the answer was not taken once its record could be reached: %+v", l)
}
asked("take-over")
})
}
b.assign("a", "mod-y")
b.put(grants.ReadAccounts, "a.mod-y", map[string]any{"user": "a.mod-y", "hash": "$2a$10$x", "generation": 1})
said = nil
res := p.Compose()
if res.Sealed == nil || !strings.Contains(res.Sealed.Why, "publishing would lose the last holder of "+seat) {
t.Errorf("a list losing a seat grant = %+v", res.Sealed)
}
if now, _, _ := p.Sealed(); now.Digest != g.Digest {
t.Errorf("a list losing a seat grant was written: %s, was %s", now.Digest, g.Digest)
}
if len(said) != 1 || said[0].Key != grants.HeldCondition || said[0].State != asks.StateRaised || !strings.Contains(said[0].Summary, "the last holder of "+seat) {
t.Errorf("the held list did not raise the warning naming the lost grant: %+v", said)
}
b.loaded = inForce
if p.Compose(); func() bool { now, _, _ := p.Sealed(); return now.Digest == g.Digest }() {
t.Errorf("a list that only adds a.mod-y was not written")
}
written, _, _ := p.Sealed()
b.mu.Lock()
b.refuseUsers = true
b.mu.Unlock()
b.mu.Lock()
b.unassign("a", "mod-y")
b.mu.Unlock()
if res := p.Compose(); res.Sealed == nil || !strings.Contains(res.Sealed.Why, "the grants of the list in force cannot be read") {
t.Errorf("a pass whose loaded list cannot be read = %+v", res.Sealed)
}
if now, _, _ := p.Sealed(); now.Digest != written.Digest {
t.Errorf("a list was written while the loaded list could not be read")
}
b.mu.Lock()
b.refuseUsers = false
b.mu.Unlock()
t.Run("only memberships published, nothing asserted", func(t *testing.T) {
for _, s := range b.publishes {
if !strings.HasPrefix(s, "mesh.assignment.") {
t.Errorf("published on %s", s)
}
t.Run("nothing published, nothing asserted", func(t *testing.T) {
if len(b.publishes) != 0 {
t.Errorf("published on %v", b.publishes)
}
for _, w := range b.writes {
if !slices.Contains(grants.States, strings.TrimSuffix(w, " delete")) {
@@ -1749,8 +1484,8 @@ func TestGrants(t *testing.T) {
}
}
for _, c := range b.calls {
if c != grants.CallUsers && c != grants.CallStreams && c != grants.CallRecordWarrant {
t.Errorf("called %s: the part calls only the bus module's read tools, and the hand-act log for a switch", c)
if c != grants.CallUsers && c != grants.CallStreams {
t.Errorf("called %s: the part calls only the bus module's read tools", c)
}
}
for _, e := range b.events {
@@ -2093,7 +1828,7 @@ func TestGrants(t *testing.T) {
// 774/L3a (novox/hq ADR 0547 §5, ADR 0561 §2): the part writes grants.<module>.<fp> in shadow, once each and only for a
// module whose accounts the vault holds a hash for, says grant-written for each, deletes one only when a module has no
// account with a hash and every read was made, and the tool says every difference from the controller's.
t.Run("grants: written in shadow once each, only for hashed accounts, deleted only when read whole, compared with the controller's; after take-over once the loaded list carries them", func(t *testing.T) {
t.Run("grants: written once each, only for hashed accounts, deleted only when read whole, and once the loaded list carries them", func(t *testing.T) {
b := fixture()
b.loaded = shadowOf(t, b)
p := partOn(b)
@@ -2162,10 +1897,6 @@ func TestGrants(t *testing.T) {
}
loads("a.nats.src2", "sha256:aa")
p.Objects = &fakeObjects{}
sw, _ := json.Marshal(grants.Switch{V: 1, State: grants.TookOver, Acts: []grants.Act{}})
if _, err := (ownStore{b}).Put(grants.StateTakeover, grants.KeySwitch, sw); err != nil {
t.Fatal(err)
}
moves := func(module, manifest, fp string) string {
digest := delivery.ManifestDigest([]byte(manifest))
b.put(grants.ReadManifests, digest, delivery.Manifest{V: delivery.Version, Module: module, JSON: manifest})
@@ -2613,10 +2344,6 @@ func TestGrants(t *testing.T) {
p.Objects = &fakeObjects{}
var said []asks.Condition
p.Say = func(c asks.Condition) error { said = append(said, c); return nil }
sw, _ := json.Marshal(grants.Switch{V: 1, State: grants.TookOver, Acts: []grants.Act{}})
if _, err := (ownStore{b}).Put(grants.StateTakeover, grants.KeySwitch, sw); err != nil {
t.Fatal(err)
}
p.Compose()
inForce, _, _ := p.Sealed()
if inForce.Digest == "" {
@@ -2640,10 +2367,6 @@ func TestGrants(t *testing.T) {
if len(said) != 1 || said[0].Key != grants.HeldCondition || said[0].State != asks.StateRaised || !strings.Contains(said[0].Summary, "a.mod-x, a.nats") {
t.Errorf("the hold was said as %+v", said)
}
// The hold is its own guard, beside take-over's refusals (ADR 0562): the switch stays taken over.
if p.SwitchState() != grants.TookOver {
t.Errorf("the hold moved the switch to %s", p.SwitchState())
}
// A person accepts exactly the named users' changes of the held list, with why: refused for another set, with no
// why, or once the list changed since; accepted, the next pass writes it once, said as a hand act.
if _, err := p.AcceptHashes("person.jochen", []string{"a.mod-x"}, "rotated by hand"); err == nil || !strings.Contains(err.Error(), "a.mod-x, a.nats") {
@@ -2787,10 +2510,6 @@ func TestGrants(t *testing.T) {
}
return nil
}
sw, _ := json.Marshal(grants.Switch{V: 1, State: grants.TookOver, Acts: []grants.Act{}})
if _, err := (ownStore{b}).Put(grants.StateTakeover, grants.KeySwitch, sw); err != nil {
t.Fatal(err)
}
p.Compose()
inForce, _, _ := p.Sealed()
if inForce.Digest == "" {
@@ -2843,17 +2562,6 @@ func TestGrants(t *testing.T) {
})
}
type fakeAsker struct {
asks []asks.Ask
wheres map[string]*asks.Warrant // the router's warrants, by ask
fail bool
cancels []string
// takeErr makes Take fail as the asker does when the part's own record cannot be reached: no handling, an error.
takeErr error
}
func (f *fakeAsker) Cancel(id string) error { f.cancels = append(f.cancels, id); return nil }
// fakeObjects is the part's object bucket of sealed lists.
type fakeObjects struct{ objects map[string][]byte }
@@ -2867,27 +2575,6 @@ func (o *fakeObjects) PutObject(name string, data []byte) error {
func (o *fakeObjects) DeleteObject(name string) error { delete(o.objects, name); return nil }
func (f *fakeAsker) Ask(a asks.Ask) error {
if f.fail {
return errors.New("the bus refused the publish")
}
f.asks = append(f.asks, a)
return nil
}
func (f *fakeAsker) Where(id string) (string, *asks.Warrant, error) { return "open", f.wheres[id], nil }
// Take hands the warrant to act as the SDK's asker does for one that chose, and records any other.
func (f *fakeAsker) Take(w asks.Warrant, act func(asks.Option, asks.Warrant) error) (asker.Handled, error) {
if f.takeErr != nil {
return "", f.takeErr
}
if w.Outcome != asks.OutcomeChosen {
return asker.Recorded, nil
}
return asker.Acted, act(asks.Option{ID: w.Option}, w)
}
func nonNil(s []string) []string {
if s == nil {
return []string{}
modules/mesh-grants/internal/grants/accounts.go
+1 -1
View File
@@ -139,7 +139,7 @@ func (p *Part) Accounts(user string) (any, error) {
answer["last_pass"] = res.Accounts
}
// And the durable consumers the accounts need, as the last pass found them: missing and differing ones are read
// here in shadow too, where no condition is raised (ADR 0561 §6, §9).
// here too (ADR 0561 §6, §9).
if res.Consumers != nil {
answer["consumers"] = res.Consumers
}
modules/mesh-grants/internal/grants/assertions.go
+2 -2
View File
@@ -50,11 +50,11 @@ type Assertion struct {
}
// ShadowAsserts is what an assertion says where nothing may be created: in the tools, which ask the part's process.
const ShadowAsserts = "in shadow: composed and compared, nothing asserted (ADR 0547 clause 5 asserts from take-over)"
const ShadowAsserts = "in the tools: composed and compared, nothing asserted (the part's process creates what the bus lacks)"
// CreatesMissing is what an assertion of the part's process says (the operator, 2026-10-26 08:54; issue 1790).
const CreatesMissing = "a bucket, log or stream the manifest declares and the bus lacks is created with the manifest's limits, once per " +
"manifest; one the bus holds is never changed or deleted, a difference is only said (ADR 0547 clause 5 asserts the rest from take-over)"
"manifest; one the bus holds is never changed or deleted, a difference is only said"
// Creator creates one stream the bus does not hold, with the configuration given, and never updates or deletes one: the
// part's process alone, whose account is granted the create of each declared stream by name (the operator, 2026-10-26
modules/mesh-grants/internal/grants/consumers.go
+4 -5
View File
@@ -14,11 +14,10 @@ import (
"git.novox.be/novox/mesh-sdk/go/buscompose"
)
// **The durable consumers an account needs** (novox/hq ADR 0561 §6, issue 774). The controller creates a module's
// consumer on EVENTS and each process's on its work queue when it issues the account. From the write that hands its
// lease away the grants part creates them, with the configuration the bus composer derives (buscompose.AccountConsumers),
// only where the bus lacks one; one the bus holds is never changed or deleted, a difference is only said. Before that
// write they are compared in shadow alone, so two parts never create one consumer.
// **The durable consumers an account needs** (novox/hq ADR 0561 §6, issue 774): a module's consumer on EVENTS and each
// process's on its work queue. Since the controller ended by its unassignment (ADR 0590, issue 1126) the grants part
// alone creates them, with the configuration the bus composer derives (buscompose.AccountConsumers), only where the bus
// lacks one; one the bus holds is never changed or deleted by a pass, a difference is only said.
// ReadLease is the controller's lease as the part reads it (buscompose.NamedReaders names the vault and the part), and
// LeaseKey its one key.
modules/mesh-grants/internal/grants/follow.go
+4 -16
View File
@@ -21,14 +21,10 @@ type Watcher interface {
// config values, the node agents' haves (a machine's first have, and the bus node's loaded list) and the assigner's
// wants (the manifest each machine is wanted to run), and the vault's hashes of the bus accounts.
//
// And its own state lease, whose key removal-due's expiry marker starts the day-later removal (ADR 0582 clause 6), and
// the seat mesh-enrolment's keys (ADR 0587 §4, §5): a key at live composes enrol.<node>, and its end, by the holder or by
// And the seat mesh-enrolment's keys (ADR 0587 §4, §5): a key at live composes enrol.<node>, and its end, by the holder or by
// the bus's expiry, is a change of the key that takes the user out; no event is needed beside the watch.
var Followed = []string{ReadModules, ReadResolved, ReadHave, ReadWants, ReadAccounts, ReadPlacement, ReadLease, ReadTakeover,
ReadEnrolments, ReadNodeLogins, OwnLease}
// OwnLease is the part's own state lease as the follower watches it, `<module>.<state>`.
const OwnLease = Module + "." + StateLease
ReadEnrolments, ReadNodeLogins}
// Follower composes again on every change it hears, never on a timer. It never stops for a refusal: a watch it cannot
// open is said, composed without, and opened again at the next wake (any change it does hear, or a reconnect).
@@ -129,14 +125,6 @@ func (f *Follower) heard(state, key string, value []byte) bool {
}
return false
}
if state == OwnLease {
// The marker of removal-due (its value gone: expired, or deleted by a hand-back, which the removal tells apart).
if key == KeyRemovalDue && value == nil && f.Part != nil {
f.Part.RemovalDueHeard()
return true
}
return false
}
if state == ReadLease {
// Renewed every few seconds by its holder: only the write that hands it away, or back, composes again (ADR 0561 §6).
// Once handed away nobody renews it, so any change is heard: it may be the lease altered (ADR 0582 clause 4).
@@ -186,8 +174,8 @@ func (f *Follower) open() {
}
if f.Hear != nil && !f.heard_ {
err := f.Hear(func(event string, body []byte) {
// An account the vault minted, or a bucket's expiry set (the lease's, altered: ADR 0582 clause 4).
if IsAccountIssued(event) || IsBucketExpirySet(event) {
// An account the vault minted.
if IsAccountIssued(event) {
f.Wake()
return
}
modules/mesh-grants/internal/grants/grants.go
+7 -9
View File
@@ -15,9 +15,8 @@ import (
// **The grants the part writes** (novox/hq ADR 0547 §4, §5, ADR 0561 §2, §6; issue 774). A module's grant is the record
// that its bus credential exists for the fingerprint the catalogue's trunk names for it: `<module>.<fingerprint hex>` in
// the part's state grants, naming the accounts (the module's own and each process's) the vault holds a hash for. It is the
// controller's grant as the controller composes it (cmd/mesh-controller grants.go grantsFrom), so the two compare by key.
// Until the controller's lease is handed away nobody reads the part's keys: the assigner reads the controller's
//, and the tool grants says every difference, so the switch is made on a measured equality.
// grant the controller composed (cmd/mesh-controller grants.go grantsFrom), whose readers it took over; the assigner reads
// the part's keys alone since the controller ended by its unassignment (ADR 0590, issue 1126).
//
// A module of which an account has no hash yet is granted the accounts that have one; a module none of whose accounts has
// one has no grant, and one it was written for is deleted (an older grant naming a node whose account is gone would let the
@@ -35,7 +34,7 @@ type GrantsWritten struct {
// `<module>: <why>`: the assigner's sixth rule wants none of it (MissingGrants). Never passed over in silence: once
// the part's grants are the ones read, it is the condition mesh-grants.grant-missing.
Missing []string `json:"missing,omitempty"`
// Waits is each grant composed and not written, `<key>: <why>`: after take-over a grant is new or changed only once the
// Waits is each grant composed and not written, `<key>: <why>`: a grant is new or changed only once the
// list the bus loaded carries it (issue 774, GrantWaits). Read with grants_grants (last_pass).
Waits []string `json:"waits,omitempty"`
}
@@ -46,9 +45,9 @@ type GrantsWritten struct {
// names): every publish and subscribe allowed, the allowance of responses, and no deny the composed account lacks. A grant
// that adds no permission is written at once, whatever list is held or pending; one that adds any waits until a list that
// carries it is loaded. loaded is the loaded list by user, nil when its contents are unknown (unknown says why): then only
// a grant naming no account is written. In shadow nobody reads the part's grants and nothing waits. Pure.
func GrantWaits(tookOver bool, accounts []string, composed, loaded map[string]User, unknown string) string {
if !tookOver || len(accounts) == 0 {
// a grant naming no account is written. Pure.
func GrantWaits(accounts []string, composed, loaded map[string]User, unknown string) string {
if len(accounts) == 0 {
return ""
}
if loaded == nil {
@@ -421,7 +420,6 @@ func (p *Part) writeGrants(res *Result, in Inputs, recs buscompose.Records, comp
done := &GrantsWritten{Wanted: len(want), Missing: MissingGrants(in, want, drop, skip)}
res.Grants = done
p.sayMissing(done.Missing)
tookOver := p.SwitchState() == TookOver
byUser := map[string]User{}
for _, u := range composed {
byUser[u.User] = u
@@ -441,7 +439,7 @@ func (p *Part) writeGrants(res *Result, in Inputs, recs buscompose.Records, comp
continue
}
}
if waits := GrantWaits(tookOver, g.Accounts, byUser, loaded, unknown); waits != "" {
if waits := GrantWaits(g.Accounts, byUser, loaded, unknown); waits != "" {
done.Waits = append(done.Waits, key+": "+waits)
continue
}
modules/mesh-grants/internal/grants/inputs.go
+11 -12
View File
@@ -1,10 +1,9 @@
// Package grants is the grants part in shadow (novox/hq ADR 0547 clauses 5 to 8, issue 1945): it composes the bus's user
// list, each module's membership and each manifest's buckets, logs and streams from what the bus holds, with the bus
// composer the controller uses (mesh-sdk buscompose), and compares them with what the bus loaded and holds. Beyond its own
// state it takes effect in two things alone, each by a grant the operator allowed: it publishes memberships (2026-10-25
// 22:31), and creates a bucket, log or stream a manifest declares that the bus lacks (2026-10-26 08:54, issue 1790). It
// changes no stream the bus holds, writes no grant and no user list the bus node reads, until the operator's take-over,
// which the operator asks on their channel and the part runs on their yes, after its refusals.
// Package grants is the grants part (novox/hq ADR 0547 clauses 5 to 8, issue 1945): it composes the bus's user list, each
// module's membership and each manifest's buckets, logs and streams from what the bus holds, with the bus composer
// (mesh-sdk buscompose), and compares them with what the bus loaded and holds. It publishes memberships (2026-10-25
// 22:31), creates a bucket, log or stream a manifest declares that the bus lacks (2026-10-26 08:54, issue 1790), writes
// the grants the assigner reads and the user list the bus node follows alone, and makes the account consumers, since the
// controller ended by its unassignment (ADR 0590, issue 1126). It changes no stream the bus holds.
package grants
import (
@@ -56,7 +55,7 @@ const (
)
// Invokes is every tool the part calls.
var Invokes = append([]string{CallUsers, CallUserCan, CallStreams, CallRootFree, CallRecordWarrant}, LeaseCalls...)
var Invokes = []string{CallUsers, CallUserCan, CallStreams, CallRootFree}
// Reader reads another part's state, named `<module>.<state>`. A refusal (a read not granted yet, a bucket not there) is
// an error the part names as a gap and composes past: it never stops for one.
@@ -74,7 +73,7 @@ type Caller interface {
type Gap struct {
Input string `json:"input"`
Why string `json:"why"`
// Failed says the gap is a read that failed this pass, not one that stands (ADR 0547 clause 7: after take-over a
// Failed says the gap is a read that failed this pass, not one that stands (ADR 0547 clause 7: a
// list composed past a failed read is not sealed).
Failed bool `json:"failed,omitempty"`
}
@@ -135,8 +134,8 @@ type Inputs struct {
Gaps []Gap
}
// HeldHash is one key of the vault's bus-accounts: the hash, and what it is of. The hash is never said or written by
// the part in shadow.
// HeldHash is one key of the vault's bus-accounts: the hash, and what it is of. The hash is never said by the part, and
// written only into the list sealed to the bus node.
type HeldHash struct {
Hash string `json:"hash"`
Generation int `json:"generation,omitempty"`
@@ -246,7 +245,7 @@ func ReadInputs(r Reader, cache *Manifests) Inputs {
nodes := map[string]bool{}
// **The seat's placement alone places modules** (novox/hq ADR 0549 §5, ADR 0567 §9; issue 774): the controller's
// assignments are read no more, and the seats held once for the mesh are held where it says. Not read is a failed
// read: nothing is placed, so after take-over the list composed past it is not sealed.
// read: nothing is placed, so the list composed past it is not sealed.
if keys, err := r.Keys(ReadPlacement); err != nil {
in.Gaps = append(in.Gaps, Gap{Input: ReadPlacement, Why: "refused or absent: nothing is placed (" + err.Error() + ")", Failed: true})
} else {
modules/mesh-grants/internal/grants/leasecheck.go
-280
View File
@@ -1,280 +0,0 @@
package grants
import (
"encoding/json"
"fmt"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// **The controller's lease, what still depends on the controller, and a dry run of the lease write's check** (issue 774,
// design 47 L3; novox/hq ADR 0547 §11, ADR 0561 §3, ADR 0571 §13). One read-only answer from bus state: the lease key,
// every manifest the indexer holds, and the bus module's streams. It writes nothing, and asks the controller nothing.
//
// It lives on the grants part because the part is where the lease write takes most of its effect and already reads all
// it needs: the lease (buscompose.NamedReaders names it), every manifest at the indexer's trunk, placement, and the bus's
// streams through the bus module's read tools. No record names the home of the lease write's check; ADR 0571 §13 names
// its test in three repositories, and this is the same judgement over the manifests the mesh holds now.
// ToolLeaseCheck is the tool's name.
const ToolLeaseCheck = "grants_lease_check"
// Controller is the module and the seat the check is about.
const Controller = "mesh-controller"
// **The judge is mesh-sdk's** (catalogue.Dependencies, catalogue.Judge; issue 774): the same judgement the three
// repositories' manifest tests run through catalogue.ManifestRefusals (ADR 0571 §13), re-exported here by its names.
const (
DependsRead = catalogue.DependsRead
DependsInvoke = catalogue.DependsInvoke
DependsConsume = catalogue.DependsConsume
DependsUses = catalogue.DependsUses
DependsWaits = catalogue.DependsWaits
PhaseBefore = catalogue.PhaseBefore
PhaseWrite = catalogue.PhaseWrite
)
type (
// Dependency, Kept, Verdict and Coverage are mesh-sdk's.
Dependency = catalogue.Dependency
Kept = catalogue.KeptDependency
Verdict = catalogue.Verdict
Coverage = catalogue.Coverage
)
// KeepList and EndedBy are mesh-sdk's named list and the records that ended what it no longer keeps.
var (
KeepList = catalogue.KeepList
EndedBy = catalogue.EndedBy
)
// Dependencies are every declaration on the controller in the manifests given (catalogue.Dependencies).
func Dependencies(manifests map[string]catalogue.Manifest) []Dependency {
return catalogue.Dependencies(manifests)
}
// Judge runs the lease write's check in one phase (catalogue.Judge).
func Judge(deps []Dependency, phase string, cov Coverage, handedAway bool, leaseSays string) Verdict {
return catalogue.Judge(deps, phase, cov, handedAway, leaseSays)
}
// LeaseNow is who holds the lease and what follows from it.
type LeaseNow struct {
HandedAway bool `json:"handed_away"`
Says string `json:"says"`
Holder *buscompose.LeaseHolder `json:"holder,omitempty"`
Since *time.Time `json:"since,omitempty"`
Does []string `json:"does"`
}
// While the controller holds the lease, and once it is handed away: what the holder and the parts watching the key do
// because of it, by the records that say so.
var (
DoesWhileController = []string{
"the controller mints every new bus account; the vault mints none and says what it holds back (ADR 0561 §3)",
"the grants part creates no durable consumer an account needs, and only compares them (ADR 0561 §6)",
"the grants part publishes memberships on the controller's assignments stream beside its own state (ADR 0571 §10)",
"mesh-healers writes each act it would take in `would`, and acts on none (ADR 0564 §8)",
"the artifact sweep lets go of nothing; the controller's take-in sweeps and says `artifacts-gone` (ADR 0571 §4)",
"the watchman only lists dead letters; the controller keeps DEAD_LETTERS through its consumer of DEAD_LETTER_NOTICES (ADR 0564 §5)",
"the controller keeps composing for the bus node, whose agent follows its user list, so the hand-backs of config, conditions, placement and the user list still answer (ADR 0547 §8, ADR 0564 §9, ADR 0565 §4, ADR 0567 §11)",
}
DoesHandedAway = []string{
"the vault mints every new bus account (ADR 0561 §1)",
"the grants part creates the durable consumers an account needs, and stops publishing on the assignments stream (ADR 0561 §6, ADR 0571 §10)",
"mesh-healers acts (ADR 0564 §8)",
"the artifact sweep sweeps and records every deletion in the hand-act log (ADR 0571 §4)",
"the watchman keeps the dead letters (ADR 0564 §5)",
"every hand-back is answered as gone (ADR 0547 §11, ADR 0564 §9, ADR 0565 §4, ADR 0567 §11)",
}
)
// Lease reads the lease key and says who holds it, since when, and what follows.
func Lease(r Reader) LeaseNow {
raw, found, err := r.Get(ReadLease, LeaseKey)
away, says := buscompose.LeaseHandedAway(raw, found, err)
out := LeaseNow{HandedAway: away, Says: says, Does: DoesWhileController}
if away {
out.Does = DoesHandedAway
}
if err == nil && found && len(raw) > 0 {
var h buscompose.LeaseHolder
if json.Unmarshal(raw, &h) == nil {
out.Holder = &h
if !h.Taken.IsZero() {
t := h.Taken
out.Since = &t
}
}
}
return out
}
// StreamUse is one of the controller's streams (its buckets, the assignments stream, the dead-letter notices) as the bus
// module answers it: who consumes it now, and what it last carried.
type StreamUse struct {
Stream string `json:"stream"`
Consumers int `json:"consumers"`
Durable []string `json:"durable,omitempty"`
Messages int64 `json:"messages"`
Subjects int64 `json:"subjects_held,omitempty"`
Last string `json:"last,omitempty"`
}
// The streams of the controller's that are not its buckets: the assignments stream it publishes memberships on, and the
// notices its consumer `controller` turns into dead letters until the lease write (ADR 0564 §5).
const (
StreamAssignments = "ASSIGNMENTS"
StreamDeadLetterNotices = "DEAD_LETTER_NOTICES"
)
// controllerStream says whether a stream is the controller's.
func controllerStream(name string) bool {
return strings.HasPrefix(name, "KV_"+Controller+"_") || name == StreamAssignments || name == StreamDeadLetterNotices
}
// Duties reads, from the bus module's streams, every stream of the controller's someone consumes, with its durable
// consumers by name (a watcher of a bucket is ephemeral and names nobody). Streams with no consumer are counted apart.
func Duties(c Caller) (used []StreamUse, idle []string, err error) {
raw, err := c.Call(CallStreams, map[string]any{})
if err != nil {
return nil, nil, fmt.Errorf("%s: %w", CallStreams, err)
}
type consumer struct {
Name string `json:"name"`
Kind string `json:"kind"`
Max int `json:"max_deliver"`
Ack string `json:"ack_policy"`
}
type stream struct {
Name string `json:"name"`
Consumers int `json:"consumers"`
Messages int64 `json:"messages"`
Held int64 `json:"subjects_held"`
Last string `json:"last"`
Detail []consumer `json:"consumer_detail"`
}
var got struct {
Detail []stream `json:"detail"`
}
if err := json.Unmarshal(unwrap(raw), &got); err != nil {
return nil, nil, fmt.Errorf("%s answered no streams: %w", CallStreams, err)
}
for _, s := range got.Detail {
if !controllerStream(s.Name) {
continue
}
if s.Consumers == 0 {
idle = append(idle, s.Name)
continue
}
u := StreamUse{Stream: s.Name, Consumers: s.Consumers, Messages: s.Messages, Subjects: s.Held, Last: s.Last}
if one, err := c.Call(CallStreams, map[string]any{"stream": s.Name}); err == nil {
var d struct {
Detail []stream `json:"detail"`
}
if json.Unmarshal(unwrap(one), &d) == nil && len(d.Detail) > 0 {
for _, cs := range d.Detail[0].Detail {
if durable(cs.Name) {
u.Durable = append(u.Durable, cs.Name)
}
}
}
}
used = append(used, u)
}
sort.Slice(used, func(i, j int) bool { return used[i].Stream < used[j].Stream })
sort.Strings(idle)
return used, idle, nil
}
// durable says whether a consumer's name is one a part gave it: the bus names an ephemeral watcher with eight random
// letters and digits, which no part of the mesh uses.
func durable(name string) bool {
if len(name) != 8 {
return true
}
for _, r := range name {
if !(r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9') {
return true
}
}
return false
}
// LeaseAnswer is the tool's answer.
type LeaseAnswer struct {
Lease LeaseNow `json:"lease"`
Dependencies []Dependency `json:"dependencies"`
// Unkept are the dependencies no record keeps, by module: what the check refuses in every phase.
Unkept map[string][]string `json:"unkept"`
Manifests int `json:"manifests_read"`
// Indexed is how many modules the indexer holds; -1 when its keys could not be listed.
Indexed int `json:"modules_indexed"`
Unread []Gap `json:"manifests_unread,omitempty"`
// Streams are the controller's streams someone consumes now; Idle those no one does.
Streams []StreamUse `json:"streams,omitempty"`
Idle []string `json:"streams_idle,omitempty"`
StreamsError string `json:"streams_unread,omitempty"`
// Check is the dry run at the write (the three alone), Before the same over the whole list before the write's
// delivery.
Check Verdict `json:"check"`
Before Verdict `json:"check_before_delivery"`
Beside []string `json:"not_judged_here"`
}
// NotJudgedHere are the lease write's other conditions, each with the tool that answers it (design 47 L3, ADR 0547 §11).
var NotJudgedHere = []string{
"L4 to L7 delivered: the issue tracker's milestones 1952, 1953, 1954 and 1955",
"every node's record trusts the vault's key: mesh-delivery-view.node_trusts (ADR 0550 §2)",
"mesh-config's compare empty and its take-over done, and mesh-wants.upgrade compare empty: mesh-config.compare, mesh-wants.upgrade (ADR 0565 §14)",
"the condition keeper's take-over done: mesh-conditions.compare (ADR 0564 §9)",
}
// LeaseCheck answers the tool: who holds the lease and what follows, every dependency on the controller with what keeps
// it, the controller's streams in use, and the lease write's check run dry. Read-only.
func (p *Part) LeaseCheck() (any, error) {
in := ReadInputs(p.Read, &p.cache)
out := LeaseAnswer{Lease: Lease(p.Read), Unkept: map[string][]string{}, Manifests: len(in.Manifests), Unread: in.Refused,
Beside: NotJudgedHere}
for _, g := range in.Gaps {
if g.Failed && strings.HasPrefix(g.Input, ReadModules) {
out.Unread = append(out.Unread, g)
}
}
out.Dependencies = Dependencies(in.Manifests)
if out.Dependencies == nil {
out.Dependencies = []Dependency{}
}
for _, d := range out.Dependencies {
if d.Kept == "" {
out.Unkept[d.Module] = append(out.Unkept[d.Module], d.Kind+" "+d.What)
}
}
if used, idle, err := Duties(p.Call); err != nil {
out.StreamsError = err.Error()
} else {
out.Streams, out.Idle = used, idle
}
// Only the indexer's own modules count as read: a manifest the mesh provides itself must not stand in for one missing.
cov := Coverage{Indexed: -1}
for _, g := range out.Unread {
cov.Unread = append(cov.Unread, catalogue.Unread{Input: g.Input, Why: g.Why})
}
if keys, err := p.Read.Keys(ReadModules); err == nil {
cov.Indexed = len(keys)
for _, k := range keys {
if _, ok := in.Manifests[k]; ok {
cov.Read++
}
}
}
out.Indexed = cov.Indexed
out.Check = Judge(out.Dependencies, PhaseWrite, cov, out.Lease.HandedAway, out.Lease.Says)
out.Before = Judge(out.Dependencies, PhaseBefore, cov, out.Lease.HandedAway, out.Lease.Says)
return out, nil
}
modules/mesh-grants/internal/grants/leasewrite.go
-962
View File
@@ -1,962 +0,0 @@
package grants
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/handacts"
"git.novox.be/novox/mesh-sdk/go/health"
)
// **The lease write, a verb of the seat mesh-grants** (novox/hq ADR 0582, issue 774). `lease-write` takes a why and is
// refused, naming the check, until every precondition of clause 2 reads green; otherwise the operator is asked once, on
// a channel that proves who answered, with the checks shown and their digest bound to the yes (clause 1). On the
// warrant the checks are read again, and then, each step read back before the next (clause 3): the controller's service
// is stopped on its node and read back stopped; the bus's user list is composed without the controller's write on the
// lease (buscompose.Records.LeaseCut) and waited on until the bus node's have says it loaded it; the lease bucket's
// expiry is removed through the bus module's tool; the key `holder` names mesh-grants with the operator, the why, the
// ask and the time; `lease-handed` is said, the act recorded through mesh-hand-acts.warranted, and `removal-due` is
// written in the state `lease` with a day to live. A lease altered after it is said as `lease-altered`, once, and
// nothing is written again until the operator asks (clause 4). `lease-hand-back` reverses it for a day (clause 5). The
// expiry marker of `removal-due`, and nothing else, starts the day-later removal of the controller's consumer of
// DEAD_LETTER_NOTICES and the stream ASSIGNMENTS (clause 6). Every step waits on an event (a warrant, the bus node's
// have, a marker), never on a timer.
// The verbs, events and states of the lease write.
const (
ActLeaseWrite = catalogue.GrantsVerbLeaseWrite
ActLeaseHandBack = catalogue.GrantsVerbLeaseHandBack
EventLeaseHanded = catalogue.GrantsEventLeaseHanded
// EventLeaseAltered and EventRemovalFailed are the seat's other two events.
EventLeaseAltered = catalogue.GrantsEventLeaseAltered
EventRemovalFailed = catalogue.GrantsEventRemovalFailed
// StateLease is the part's state with a time to live per key, holding KeyRemovalDue alone.
StateLease = "lease"
KeyRemovalDue = "removal-due"
// KeyLease is the lease write's record in StateTakeover: the cut, the write, the removal and every act.
KeyLease = "lease"
// ToolRemovalDue is the tool that writes removal-due again with a short time to live, the recovery for a marker
// missed while the part was down.
ToolRemovalDue = "grants_removal_due"
)
// RemovalAfter is how long after the write the controller's consumer and stream are removed (ADR 0582 clause 6).
const RemovalAfter = 24 * time.Hour
// The calls the lease write makes, as the manifest's invokes name them.
const (
// CallServiceStop, CallServiceStart and CallServiceStatus are the node seat's verbs, asked of the controller's node.
CallServiceStop = "seat:" + catalogue.ServiceManagerSeat + ".stop"
CallServiceStart = "seat:" + catalogue.ServiceManagerSeat + ".start"
CallServiceStatus = "seat:" + catalogue.ServiceManagerSeat + ".status"
// CallBucketExpiry sets and restores a bucket's expiry, recording the act itself; CallConsumerRemove and
// CallStreamRemove are the removal's two tools.
CallBucketExpiry = "nats.nats_bucket_expiry"
CallConsumerRemove = "nats.nats_consumer_remove"
CallStreamRemove = "nats.nats_stream_remove"
// CallRecordHandAct records the removals as hand acts (ADR 0582 clause 6, ADR 0565 §11).
CallRecordHandAct = "seat:" + handacts.Seat + ".hand-act"
// The checks' calls (clause 2).
CallNodeTrusts = "mesh-delivery-view.node_trusts"
CallConfigCompare = "seat:mesh-config.compare"
CallUpgrade = "seat:" + catalogue.WantsSeat + ".upgrade"
CallVaultStatus = "mesh-vault.vault_status"
CallIssue = "seat:issue-tracker.get"
)
// LeaseCalls are every call the lease write makes.
var LeaseCalls = []string{CallServiceStop, CallServiceStart, CallServiceStatus, CallBucketExpiry, CallConsumerRemove,
CallStreamRemove, CallRecordHandAct, CallNodeTrusts, CallConfigCompare, CallUpgrade, CallVaultStatus, CallIssue}
// ControllerUnit is the controller's service on its node.
const ControllerUnit = Controller + ".service"
// LeaseExpiry is the lease bucket's expiry a hand-back restores (buscompose.LeaseTTL).
var LeaseExpiry = buscompose.LeaseTTL
// RemovedTools are the bus module's tools the lease write and its removal need: the check reads them in its manifest.
var RemovedTools = []string{"nats_bucket_expiry", "nats_consumer_remove", "nats_stream_remove"}
// EarlierSteps are the issues of design 47's earlier steps the write waits on (ADR 0582 clause 2: issues 1952-1955 delivered).
var EarlierSteps = []uint64{1952, 1953, 1954, 1955}
// LeaseBus is the process's hand on the controller's lease: the key `holder` put and deleted, and the bucket's expiry
// read. Nil in the tools, which ask the process.
type LeaseBus interface {
PutHolder(value []byte) error
DeleteHolder() error
Expiry() (time.Duration, error)
}
// TTLStore is a Store that writes a key with its own time to live (the state lease).
type TTLStore interface {
PutTTL(state, key string, value []byte, ttl time.Duration) (uint64, error)
}
// SeatSayer says one of the seat's events, `mesh.seat.mesh-grants.event.<event>`.
type SeatSayer func(event string, body any) error
// Check is one precondition as read: its name, whether it reads green, and what it says.
type Check struct {
Name string `json:"name"`
Green bool `json:"green"`
Says string `json:"says"`
}
// checksDigest is the digest of the checks as shown: what the yes binds.
func checksDigest(cs []Check) string {
raw, _ := json.Marshal(cs)
sum := sha256.Sum256(raw)
return "sha256:" + hex.EncodeToString(sum[:])
}
// LeaseAct is one lease-write or lease-hand-back: asked, answered and each step it reached.
type LeaseAct struct {
Act string `json:"act"`
By string `json:"by"`
Why string `json:"why"`
At time.Time `json:"at"`
Ask string `json:"ask"`
// Checks are the preconditions as the ask showed them, and Digest their digest the yes binds.
Checks []Check `json:"checks,omitempty"`
Digest string `json:"digest,omitempty"`
Raised bool `json:"raised,omitempty"`
// Step is where it stands: asked; then on the warrant stopped, cut, loaded, expiry, written, done for a write and
// restored, uncut, done for a hand-back; or declined, expired, refused, failed.
Step string `json:"step"`
Said string `json:"said,omitempty"`
// List is the digest of the list composed at the cut (or without it, for a hand-back), awaited loaded.
List string `json:"list,omitempty"`
Done time.Time `json:"done,omitzero"`
}
// The steps an act reaches.
const (
StepAsked = "asked"
StepStopped = "stopped"
StepCut = "cut"
StepWritten = "written"
StepRestored = "restored"
StepUncut = "uncut"
StepDone = "done"
StepDeclined = "declined"
StepExpired = "expired"
StepRefused = "refused"
StepFailed = "failed"
)
// LeaseRecord is the key `lease` of StateTakeover.
type LeaseRecord struct {
V int `json:"v"`
// Cut composes the list with buscompose.Records.LeaseCut: from the write's second step until a hand-back's.
Cut bool `json:"cut"`
// Node is the controller's node, from the lease's holder at the write.
Node string `json:"node,omitempty"`
// Handed is when the key holder was written, zero when not (or handed back); Ask the write's ask.
Handed time.Time `json:"handed,omitzero"`
Ask string `json:"ask,omitempty"`
// Removed is when the day-later removal was done; zero until then.
Removed time.Time `json:"removed,omitzero"`
// Altered is what lease-altered said, empty while the lease reads as written.
Altered string `json:"altered,omitempty"`
// Open is the act asked or under way.
Open *LeaseAct `json:"open,omitempty"`
// Acts are the acts ended, newest last.
Acts []LeaseAct `json:"acts"`
}
func (p *Part) readLease() (LeaseRecord, error) {
r := LeaseRecord{V: 1, Acts: []LeaseAct{}}
raw, _, found, err := p.Store.Get(StateTakeover, KeyLease)
if err != nil {
return r, fmt.Errorf("the lease write's record cannot be read: %w", err)
}
if found {
if err := json.Unmarshal(raw, &r); err != nil {
return r, fmt.Errorf("the lease write's record does not read: %w", err)
}
}
return r, nil
}
func (p *Part) writeLease(r LeaseRecord) error {
if len(r.Acts) > keptActs {
r.Acts = r.Acts[len(r.Acts)-keptActs:]
}
if r.Acts == nil {
r.Acts = []LeaseAct{}
}
data, _ := json.Marshal(r)
if _, err := p.Store.Put(StateTakeover, KeyLease, data); err != nil {
return fmt.Errorf("the lease write's record cannot be written: %w", err)
}
return nil
}
// leaseCut says whether the list is composed with the cut.
func (p *Part) leaseCut() bool {
r, err := p.readLease()
return err == nil && r.Cut
}
// leaseHanded says whether the key holder names mesh-grants by the part's own write.
func (p *Part) leaseHanded() bool {
r, err := p.readLease()
return err == nil && !r.Handed.IsZero()
}
// end closes the open act with its step and why.
func (r *LeaseRecord) end(step, said string, at time.Time) {
if r.Open == nil {
return
}
a := *r.Open
a.Step, a.Said, a.Done = step, said, at
r.Acts = append(r.Acts, a)
r.Open = nil
}
// LeaseCheckNames are the checks LeaseChecks reads, in its order.
var LeaseCheckNames = []string{"lease-check", "earlier-steps", "node-trusts", "config-compare", "upgrade-compare",
"keeper-take-over", "vault-minting", "bus-tools"}
// LeaseChecks reads every precondition of the write (ADR 0582 clause 2), each named, in one order.
func (p *Part) LeaseChecks() []Check {
var out []Check
add := func(name string, green bool, says string) {
out = append(out, Check{Name: name, Green: green, Says: says})
}
// The lease check at the write, with the three kept to it alone (ADR 0571 §13).
if raw, err := p.LeaseCheck(); err != nil {
add("lease-check", false, "the lease check could not run: "+err.Error())
} else if a, ok := raw.(LeaseAnswer); !ok {
add("lease-check", false, "the lease check answered nothing it reads")
} else {
add("lease-check", a.Check.Passes, strings.Join(a.Check.Reasons, "; "))
}
// Design 47's earlier steps delivered.
var open []string
for _, n := range EarlierSteps {
raw, err := p.Call.Call(CallIssue, map[string]any{"number": n, "brief": true})
var is struct {
Status string `json:"status"`
}
switch {
case err != nil:
open = append(open, fmt.Sprintf("issue %d cannot be read (%v)", n, err))
case json.Unmarshal(unwrap(raw), &is) != nil || is.Status != "resolved":
open = append(open, fmt.Sprintf("issue %d is %q, not resolved", n, is.Status))
}
}
add("earlier-steps", len(open) == 0, said(open, "issues 1952 to 1955 are resolved"))
// Every node trusts the vault's key.
if raw, err := p.Call.Call(CallNodeTrusts, map[string]any{}); err != nil {
add("node-trusts", false, "node_trusts cannot be read: "+err.Error())
} else {
var t struct {
Every bool `json:"every"`
NotYet []string `json:"not_yet"`
}
_ = json.Unmarshal(unwrap(raw), &t)
add("node-trusts", t.Every, said(prefixed("not yet: ", t.NotYet), "every node's record trusts the vault's key"))
}
// mesh-config's compare empty and its take-over done.
if raw, err := p.Call.Call(CallConfigCompare, map[string]any{}); err != nil {
add("config-compare", false, "mesh-config's compare cannot be read: "+err.Error())
} else {
var c struct {
Source string `json:"source"`
Count int `json:"count"`
}
_ = json.Unmarshal(unwrap(raw), &c)
add("config-compare", c.Count == 0 && c.Source == "indexer",
fmt.Sprintf("mesh-config resolves from %q with %d differences", c.Source, c.Count))
}
// The upgrade compare empty.
if raw, err := p.Call.Call(CallUpgrade, map[string]any{"compare": "true"}); err != nil {
add("upgrade-compare", false, "mesh-wants' upgrade compare cannot be read: "+err.Error())
} else {
var u struct {
Empty bool `json:"empty"`
Splits []json.RawMessage `json:"splits"`
}
_ = json.Unmarshal(unwrap(raw), &u)
add("upgrade-compare", u.Empty, fmt.Sprintf("%d policy splits", len(u.Splits)))
}
// The condition keeper's take-over done.
raw, found, err := p.Read.Get(ReadTakeover, health.TakeoverKey)
t, terr := health.DecodeTakeover(raw, found)
switch {
case err != nil:
add("keeper-take-over", false, "the keeper's take-over cannot be read: "+err.Error())
case terr != nil:
add("keeper-take-over", false, "the keeper's take-over does not read: "+terr.Error())
default:
add("keeper-take-over", t.Follow != health.FollowController, "conditions follow "+t.Follow)
}
// The vault minting with nothing held back.
if raw, err := p.Call.Call(CallVaultStatus, map[string]any{}); err != nil {
add("vault-minting", false, "vault_status cannot be read: "+err.Error())
} else {
var v struct {
Ready bool `json:"ready"`
Minting struct {
On bool `json:"on"`
HeldBack map[string]string `json:"held_back"`
} `json:"minting"`
}
_ = json.Unmarshal(unwrap(raw), &v)
add("vault-minting", v.Ready && v.Minting.On && len(v.Minting.HeldBack) == 0,
fmt.Sprintf("ready %v, minting %v, %d held back", v.Ready, v.Minting.On, len(v.Minting.HeldBack)))
}
// The bus module's tools for the expiry and the removal live: its manifest at the indexer's trunk lists them.
in := ReadInputs(p.Read, &p.cache)
var missing []string
for _, tool := range RemovedTools {
if !slices.Contains(in.Manifests[BusModule].Tools, tool) {
missing = append(missing, tool)
}
}
add("bus-tools", len(missing) == 0, said(prefixed("the bus module lists no ", missing), "the bus module lists its three tools"))
return out
}
func said(problems []string, ok string) string {
if len(problems) == 0 {
return ok
}
return strings.Join(problems, "; ")
}
func prefixed(prefix string, items []string) []string {
if len(items) == 0 {
return nil
}
return []string{prefix + strings.Join(items, ", ")}
}
// notGreen is every check not green, `<name>: <says>`.
func notGreen(cs []Check) []string {
var out []string
for _, c := range cs {
if !c.Green {
out = append(out, c.Name+": "+c.Says)
}
}
return out
}
// changedChecks names each check whose reading differs from the one shown.
func changedChecks(shown, now []Check) []string {
was := map[string]Check{}
for _, c := range shown {
was[c.Name] = c
}
var out []string
for _, c := range now {
if w, ok := was[c.Name]; !ok || w != c {
out = append(out, c.Name+" (now: "+c.Says+")")
}
delete(was, c.Name)
}
for name := range was {
out = append(out, name+" (no longer read)")
}
slices.Sort(out)
return out
}
// LeaseWrite is the verb lease-write: refused, naming each check not green, or asked of the operator by the part's process.
func (p *Part) LeaseWrite(caller, why string) (any, error) {
return p.leaseRequest(ActLeaseWrite, caller, why)
}
// LeaseHandBack is the verb lease-hand-back.
func (p *Part) LeaseHandBack(caller, why string) (any, error) {
return p.leaseRequest(ActLeaseHandBack, caller, why)
}
func (p *Part) leaseRequest(act, caller, why string) (any, error) {
if strings.TrimSpace(why) == "" {
return nil, fmt.Errorf("%s needs a why, shown to the operator and kept", act)
}
r, err := p.readLease()
if err != nil {
return nil, err
}
if r.Open != nil {
return nil, fmt.Errorf("%s is already under way: the ask %s stands at %s", r.Open.Act, r.Open.Ask, r.Open.Step)
}
var checks []Check
switch act {
case ActLeaseWrite:
if !r.Handed.IsZero() && r.Altered == "" {
return nil, errors.New("the lease is handed to mesh-grants already: nothing to write")
}
checks = p.LeaseChecks()
if bad := notGreen(checks); len(bad) > 0 {
return nil, fmt.Errorf("lease-write is refused until every check reads green (ADR 0582 clause 2): %s", strings.Join(bad, "; "))
}
case ActLeaseHandBack:
if refused := handBackRefused(r); refused != "" {
return nil, errors.New(refused)
}
}
a := LeaseAct{Act: act, By: caller, Why: why, At: p.now(), Checks: checks, Digest: checksDigest(checks), Step: StepAsked}
a.Ask = act + "-" + p.now().Format("20060102t150405") + "-" + p.nonce()
r.Open = &a
if err := p.writeLease(r); err != nil {
return nil, err
}
_, _ = p.AskCompose(caller)
return map[string]any{"asked": a.Ask, "checks": checks, "says": "the part's process asks the operator on their channel; " +
"only the operator's yes runs it, after the checks are read again. grants_lease_check shows the lease, and " +
"the record is the key lease of the state takeover"}, nil
}
// handBackRefused is why a hand-back cannot be asked, or empty.
func handBackRefused(r LeaseRecord) string {
switch {
case !r.Removed.IsZero():
return "lease-hand-back is refused: the day-later removal ran, so the controller would start without its " +
"consumer of DEAD_LETTER_NOTICES and the stream ASSIGNMENTS (ADR 0582 clause 5)"
case r.Handed.IsZero() && !r.Cut:
return "lease-hand-back is refused: the lease is not handed to mesh-grants, so there is nothing to hand back"
}
return ""
}
// EventBucketExpirySet is the bus module's event of a bucket's expiry set, which the part consumes: a lease bucket's
// expiry changed is heard, never polled.
const EventBucketExpirySet = BusModule + ".bucket-expiry-set"
// IsBucketExpirySet says an event heard is the bus module's bucket-expiry-set.
func IsBucketExpirySet(event string) bool {
return event == EventBucketExpirySet || event == "mesh.mod."+BusModule+".event.bucket-expiry-set"
}
// isLeaseAsk says an ask is the lease write's.
func isLeaseAsk(id string) bool {
return strings.HasPrefix(id, ActLeaseWrite+"-") || strings.HasPrefix(id, ActLeaseHandBack+"-")
}
// leaseAskOf is the ask of an act, in the words the operator reads: what it does, the checks it showed, and the one
// option whose yes binds the act, the holder, the ask and the checks' digest (ADR 0582 clause 1).
func (p *Part) leaseAskOf(a LeaseAct) (asks.Ask, error) {
act := asks.Act{"verb": a.Act, "module": Module, "ask": a.Ask, "holder": Module, "checks": a.Digest}
bind, err := asks.ActDigest(act)
if err != nil {
return asks.Ask{}, err
}
var lines []string
for _, c := range a.Checks {
lines = append(lines, "✓ "+c.Name+": "+c.Says)
}
headline, does, stays := "Hand the controller's lease to mesh-grants?",
"The controller is stopped and the lease is handed to mesh-grants for good; hand-back stays open for a day.",
"The controller keeps its lease."
expl := "Every check of the lease write reads green. Your yes stops the controller, takes from it the right to write " +
"its lease, removes the lease's expiry and names mesh-grants as its holder. A controller started later fails on the lease."
if a.Act == ActLeaseHandBack {
headline, does, stays = "Give the controller its lease back?",
"The lease's expiry is restored, mesh-grants lets go of it, the controller gets its rights back and is started.",
"The lease stays with mesh-grants."
expl = "The lease was handed to mesh-grants less than a day ago. Your yes gives it back and starts the controller."
}
return asks.Ask{ID: a.Ask, Headline: headline, Explanation: expl, Who: asks.Operator,
Options: []asks.Option{{ID: "yes", Label: "Yes", Does: does, Level: asks.Approve, Binds: bind},
{ID: "no", Label: "No, leave it", Does: stays, Level: asks.Acknowledge}},
Expires: a.At.Add(asks.ApproveLasts - time.Minute), OnExpiry: stays + " Nothing changes.", About: "mesh-grants-" + a.Act,
Details: "Asked by " + a.By + ": " + a.Why + "\n" + strings.Join(lines, "\n")}, nil
}
// raiseLease asks the operator the open act once; closes one past its expiry; catches up a warrant the process missed.
func (p *Part) raiseLease() error {
if p.Asker == nil {
return nil
}
r, err := p.readLease()
if err != nil || r.Open == nil || r.Open.Step != StepAsked {
return err
}
a := r.Open
if !a.Raised {
ask, err := p.leaseAskOf(*a)
if err != nil {
return err
}
if err := p.Asker.Ask(ask); err != nil {
if p.Forget != nil {
_ = p.Forget(a.Ask)
}
_ = p.Asker.Cancel(a.Ask)
r.end(StepRefused, "the operator could not be asked: "+err.Error(), p.now())
_ = p.writeLease(r)
return fmt.Errorf("the operator could not be asked the %s: %v", a.Act, err)
}
a.Raised = true
return p.writeLease(r)
}
_, w, err := p.Asker.Where(a.Ask)
if err != nil {
return fmt.Errorf("the router's record of the ask %s cannot be read: %v", a.Ask, err)
}
if w != nil {
h, herr := p.Asker.Take(*w, p.Warranted)
if h == "" && herr != nil {
return fmt.Errorf("the operator's answer to %s could not be taken yet: %v", a.Ask, herr)
}
p.Said(h, *w, herr)
return nil
}
if p.now().After(a.At.Add(asks.ApproveLasts + askMargin)) {
r.end(StepExpired, "the ask expired: nothing was written", p.now())
return p.writeLease(r)
}
return nil
}
// leaseWarranted is the operator's answer to a lease act: a no or a refusal writes nothing; a yes runs the act.
func (p *Part) leaseWarranted(o asks.Option, w asks.Warrant) error {
r, err := p.readLease()
if err != nil {
return err
}
if r.Open == nil || r.Open.Ask != w.Ask || r.Open.Step != StepAsked {
return fmt.Errorf("the warrant answers the ask %s, which is not the lease act asked: nothing is written", w.Ask)
}
if o.ID != "yes" {
r.end(StepDeclined, "the operator said no: nothing was written", p.now())
return p.writeLease(r)
}
a := r.Open
if a.Act == ActLeaseWrite {
// Read again on the warrant (ADR 0582 clauses 1, 2): the yes binds the checks shown, so any check that reads
// otherwise now (one no longer green among them) refuses it, named.
if now := p.LeaseChecks(); checksDigest(now) != a.Digest {
r.end(StepRefused, "refused on the warrant, the checks changed since the ask: "+strings.Join(changedChecks(a.Checks, now), "; "), p.now())
p.sayLeaseRefused(a.Act, r.Acts[len(r.Acts)-1].Said)
return p.writeLease(r)
}
} else if refused := handBackRefused(r); refused != "" {
r.end(StepRefused, refused, p.now())
return p.writeLease(r)
}
a.Step = "warranted"
if err := p.writeLease(r); err != nil {
return err
}
return p.advanceLease()
}
// sayLeaseRefused tells the operator a yes was refused, as the take-over's refusal is said.
func (p *Part) sayLeaseRefused(act, why string) {
if p.Say == nil {
return
}
_ = p.Say(asks.Condition{Key: "mesh-grants." + act + "-refused", Severity: asks.SeverityWarning,
Subject: asks.Scope{Scope: "module", ID: Module}, State: asks.StateRaised, Summary: act + " refused: " + why,
Headline: "The lease write was refused", Needs: "a look at why",
Explanation: asks.NeedsYou + "a look at why. You said yes and it was refused: " + why, Resolved: "the act is asked again and goes through"})
}
// advanceLease takes the open act as far as it can now: each step read back before the next, and a step that waits (the
// list loaded) left for the pass that hears the bus node's have. A step that fails ends the act there, saying why, and
// writes nothing more.
func (p *Part) advanceLease() error {
r, err := p.readLease()
if err != nil || r.Open == nil || r.Open.Step == StepAsked {
return err
}
a := r.Open
fail := func(why string) error {
if p.Logf != nil {
p.Logf("%s failed at its step %s: %s", a.Act, a.Step, why)
}
r.end(StepFailed, why, p.now())
p.sayLeaseRefused(a.Act, why)
return p.writeLease(r)
}
if a.Act == ActLeaseHandBack {
return p.advanceHandBack(&r, fail)
}
switch a.Step {
case "warranted":
// 1. Stop the controller's service on its node, read back stopped.
node, why := p.controllerNode()
if node == "" {
return fail(why)
}
if stopped, why := p.service(CallServiceStop, node); !stopped {
return fail("the controller's service on " + node + " did not read back stopped (" + why + "): nothing more is written")
}
r.Node, a.Step = node, StepStopped
fallthrough
case StepStopped:
// 2. Compose the list without the controller's write on the lease.
r.Cut, a.Step = true, StepCut
if err := p.writeLease(r); err != nil {
return err
}
res := p.composeQuiet()
g, _, _ := p.Sealed()
if res.Sealed == nil || g.Digest == "" {
return fail("the list composed with the cut could not be written sealed: " + strings.Join(res.Errors, "; "))
}
if r, err = p.readLease(); err != nil || r.Open == nil {
return err
}
a = r.Open
a.List = g.Digest
if err := p.writeLease(r); err != nil {
return err
}
fallthrough
case StepCut:
// 3. Wait until the bus node's have says it loaded that list.
if !p.loaded(a.List) {
return nil
}
// 4. Remove the lease bucket's expiry.
if ok, why := p.setExpiry("none", *a); !ok {
return fail("the lease's expiry was not removed (" + why + "): the holder is not written")
}
// 5. Create the key holder naming mesh-grants.
h := buscompose.LeaseHolder{HandedTo: Module, HandedBy: "operator", Why: a.Why, Ask: a.Ask, At: p.now(), Host: r.Node}
raw, _ := json.Marshal(h)
if p.Lease == nil {
return fail("the part holds no hand on the lease here")
}
if err := p.Lease.PutHolder(raw); err != nil {
return fail("the key holder could not be written: " + err.Error())
}
r.Handed, r.Ask, r.Altered, a.Step = p.now(), a.Ask, "", StepWritten
if err := p.writeLease(r); err != nil {
return err
}
// 6. Say lease-handed, record the act, and write removal-due with a day to live.
p.saySeat(EventLeaseHanded, map[string]any{"holder": Module, "ask": a.Ask, "why": a.Why, "node": r.Node, "at": r.Handed})
p.recordWarrant(a.Ask)
if err := p.putRemovalDue(RemovalAfter); err != nil {
r.end(StepDone, "the lease is handed; removal-due could not be written ("+err.Error()+"): grants_removal_due writes it", p.now())
} else {
r.end(StepDone, "the lease is handed to mesh-grants; the removal is due in a day", p.now())
}
return p.writeLease(r)
}
return nil
}
// advanceHandBack restores the expiry, deletes holder and removal-due, composes without the cut, and once that list is
// loaded starts the controller (ADR 0582 clause 5).
func (p *Part) advanceHandBack(r *LeaseRecord, fail func(string) error) error {
a := r.Open
switch a.Step {
case "warranted":
if ok, why := p.setExpiry(LeaseExpiry.String(), *a); !ok {
return fail("the lease's expiry was not restored (" + why + "): the lease stays with mesh-grants")
}
if p.Lease == nil {
return fail("the part holds no hand on the lease here")
}
// The record first: the delete of removal-due is then heard as no removal (Handed is zero).
r.Handed, a.Step = time.Time{}, StepRestored
if err := p.writeLease(*r); err != nil {
return err
}
if err := p.Lease.DeleteHolder(); err != nil {
return fail("the key holder could not be deleted: " + err.Error())
}
if err := p.Store.Delete(StateLease, KeyRemovalDue); err != nil {
return fail("removal-due could not be deleted: " + err.Error())
}
r.Cut, a.Step = false, StepUncut
if err := p.writeLease(*r); err != nil {
return err
}
p.composeQuiet()
g, _, _ := p.Sealed()
fresh, err := p.readLease()
if err != nil || fresh.Open == nil {
return err
}
*r = fresh
a = r.Open
a.List = g.Digest
if err := p.writeLease(*r); err != nil {
return err
}
fallthrough
case StepUncut:
if !p.loaded(a.List) {
return nil
}
if started, why := p.service(CallServiceStart, r.Node); !started {
return fail("the controller's service on " + r.Node + " did not read back started (" + why + ")")
}
p.recordWarrant(a.Ask)
r.end(StepDone, "the lease is given back; the controller is started", p.now())
return p.writeLease(*r)
}
return nil
}
// controllerNode is the node placement places mesh-controller on (issue 774). Never the lease holder's host: that is
// the machine's hostname, and no node need be named so, so a seat verb addressed to it reaches nobody. None, or more
// than one, is refused by name and nothing is stopped.
func (p *Part) controllerNode() (string, string) {
if raw, found, err := p.Read.Get(ReadLease, LeaseKey); err == nil && found {
var h buscompose.LeaseHolder
if json.Unmarshal(raw, &h) == nil && h.HandedTo != "" {
return "", "the lease is handed to " + h.HandedTo + ", not the controller's: nothing is stopped"
}
}
in := ReadInputs(p.Read, &p.cache)
var nodes []string
for node, modules := range in.Assigned {
if slices.Contains(modules, Controller) {
nodes = append(nodes, node)
}
}
slices.Sort(nodes)
switch len(nodes) {
case 0:
return "", "no node places " + Controller + ", so the controller's node is not known: nothing is stopped"
case 1:
return nodes[0], ""
}
return "", "two or more nodes place " + Controller + " (" + strings.Join(nodes, ", ") + "), so the controller's node is not one: nothing is stopped"
}
// service asks the node seat's verb of the controller's unit and reads it back with status: stopped for a stop, started
// for a start.
func (p *Part) service(verb, node string) (bool, string) {
unit := map[string]any{"unit": ControllerUnit}
if _, err := p.Call.Call(verb+"@"+node, unit); err != nil {
return false, err.Error()
}
raw, err := p.Call.Call(CallServiceStatus+"@"+node, unit)
if err != nil {
return false, "its status cannot be read: " + err.Error()
}
var st struct {
Active string `json:"ActiveState"`
}
_ = json.Unmarshal(unwrap(raw), &st)
if verb == CallServiceStart {
return st.Active == "active", "it reads " + st.Active
}
return st.Active == "inactive" || st.Active == "failed", "it reads " + st.Active
}
// loaded says the bus node's have names the list of that digest loaded from the grants part's key.
func (p *Part) loaded(digest string) bool {
l := ReadInputs(p.Read, &p.cache).Loaded
return digest != "" && l != nil && l.Source == delivery.GrantsUserListSource && l.Digest == digest
}
// setExpiry asks the bus module's tool to set the lease bucket's expiry, and reads it back from the bucket.
func (p *Part) setExpiry(maxAge string, a LeaseAct) (bool, string) {
raw, err := p.Call.Call(CallBucketExpiry, map[string]any{"bucket": buscompose.LeaseBucket, "max_age": maxAge,
"why": a.Act + ": " + a.Why, "warrant": a.Ask})
if err != nil {
return false, err.Error()
}
var set struct {
Now string `json:"now"`
}
_ = json.Unmarshal(unwrap(raw), &set)
if set.Now != maxAge {
return false, "the tool answered " + set.Now
}
if p.Lease != nil {
got, err := p.Lease.Expiry()
want, _ := time.ParseDuration(strings.Replace(maxAge, "none", "0s", 1))
if err != nil || got != want {
return false, fmt.Sprintf("the bucket reads back %s (%v)", got, err)
}
}
return true, ""
}
// putRemovalDue writes removal-due with its time to live.
func (p *Part) putRemovalDue(ttl time.Duration) error {
s, ok := p.Store.(TTLStore)
if !ok {
return errors.New("the part's store writes no time to live here")
}
raw, _ := json.Marshal(map[string]any{"due": p.now().Add(ttl), "written": p.now()})
_, err := s.PutTTL(StateLease, KeyRemovalDue, raw, ttl)
return err
}
// saySeat says one of the seat's events; a failure is logged, never the end of the act.
func (p *Part) saySeat(event string, body any) {
if p.SaySeat == nil {
return
}
if err := p.SaySeat(event, body); err != nil && p.Logf != nil {
p.Logf("the event %s could not be said: %v", event, err)
}
}
// leaseStep is the pass's part in the lease write: the open act asked or advanced, the lease watched once handed, and
// the removal run when its marker was heard.
func (p *Part) leaseStep(res *Result) {
for _, step := range []func() error{p.raiseLease, p.advanceLease, p.watchLease} {
if err := step(); err != nil {
res.Errors = append(res.Errors, err.Error())
}
}
if p.removalHeard.Swap(false) {
if err := p.removal(); err != nil {
res.Errors = append(res.Errors, err.Error())
}
}
}
// RemovalDueHeard is the follower's: the marker of removal-due was heard (its delete, by expiry or by a hand-back).
func (p *Part) RemovalDueHeard() { p.removalHeard.Store(true) }
// watchLease says lease-altered once when the key or the bucket's expiry differs from what the part wrote (ADR 0582
// clause 4); a hand-back under way is the part's own change and is not watched.
func (p *Part) watchLease() error {
r, err := p.readLease()
if err != nil || r.Handed.IsZero() || r.Altered != "" || (r.Open != nil && r.Open.Act == ActLeaseHandBack && r.Open.Step != StepAsked) {
return err
}
var what []string
raw, found, err := p.Read.Get(ReadLease, LeaseKey)
var h buscompose.LeaseHolder
switch {
case err != nil:
return nil // not read is no change: the next pass reads it again
case !found:
what = append(what, "the key holder is gone")
case json.Unmarshal(raw, &h) != nil || h.HandedTo != Module || h.Ask != r.Ask:
what = append(what, "the key holder names "+firstOf(h.HandedTo, h.Instance, "something else")+", not mesh-grants by "+r.Ask)
}
if p.Lease != nil {
if d, err := p.Lease.Expiry(); err == nil && d != 0 {
what = append(what, "the lease's bucket expires after "+d.String())
}
}
if len(what) == 0 {
return nil
}
r.Altered = strings.Join(what, "; ")
p.saySeat(EventLeaseAltered, map[string]any{"altered": r.Altered, "ask": r.Ask, "at": p.now()})
return p.writeLease(r)
}
func firstOf(s ...string) string {
for _, x := range s {
if x != "" {
return x
}
}
return ""
}
// removal is the day-later removal, started by the marker of removal-due alone (ADR 0582 clause 6): when the lease still
// names mesh-grants, the controller's consumer of DEAD_LETTER_NOTICES and the stream ASSIGNMENTS are removed, each read
// back by the bus module's tool and recorded as a hand act. A failure is said as removal-failed and removal-due is
// written again with a day.
func (p *Part) removal() error {
r, err := p.readLease()
if err != nil || r.Handed.IsZero() || !r.Removed.IsZero() {
return err // handed back (a hand-back deletes the key), or done already
}
if raw, found, err := p.Read.Get(ReadLease, LeaseKey); err != nil || !found {
return p.removalFailed(r, "the lease cannot be read, so it is not known to name mesh-grants")
} else if away, _ := buscompose.LeaseHandedAway(raw, true, nil); !away {
return p.removalFailed(r, "the lease no longer names mesh-grants")
}
why := "a day after the lease write " + r.Ask + " (ADR 0582 clause 6)"
var failed []string
if raw, err := p.Call.Call(CallConsumerRemove, map[string]any{"stream": buscompose.DeadLetterNoticesStream,
"consumer": buscompose.ControllerName, "why": why}); err != nil && !strings.Contains(err.Error(), "has no consumer") {
failed = append(failed, "the consumer controller of DEAD_LETTER_NOTICES: "+err.Error())
} else if err == nil && !removed(raw) {
failed = append(failed, "the consumer controller of DEAD_LETTER_NOTICES does not read back removed")
} else {
p.recordHandAct("removed the controller's consumer of DEAD_LETTER_NOTICES", why)
}
if raw, err := p.Call.Call(CallStreamRemove, map[string]any{"stream": buscompose.AssignmentsStream, "why": why}); err != nil {
failed = append(failed, "the stream ASSIGNMENTS: "+err.Error())
} else if !removed(raw) {
failed = append(failed, "the stream ASSIGNMENTS does not read back removed")
} else {
p.recordHandAct("removed the stream ASSIGNMENTS", why)
}
if len(failed) > 0 {
return p.removalFailed(r, strings.Join(failed, "; "))
}
r.Removed = p.now()
return p.writeLease(r)
}
// removed reads a removal tool's answer: removed now, or not there to remove.
func removed(raw json.RawMessage) bool {
var a struct {
Removed bool `json:"removed"`
Present *bool `json:"present"`
}
_ = json.Unmarshal(unwrap(raw), &a)
return a.Removed || (a.Present != nil && !*a.Present)
}
func (p *Part) removalFailed(r LeaseRecord, why string) error {
p.saySeat(EventRemovalFailed, map[string]any{"failed": why, "ask": r.Ask, "at": p.now()})
if err := p.putRemovalDue(RemovalAfter); err != nil {
return fmt.Errorf("the removal failed (%s) and removal-due could not be written again: %v", why, err)
}
return fmt.Errorf("the day-later removal failed, tried again in a day: %s", why)
}
// recordHandAct records one removal in the hand-act log; said, never fatal.
func (p *Part) recordHandAct(what, why string) {
if p.Call == nil {
return
}
if _, err := p.Call.Call(CallRecordHandAct, map[string]any{"what": what, "why": why, "cause": "lease-removal"}); err != nil && p.Logf != nil {
p.Logf("%s, and the hand act could not be recorded: %v", what, err)
}
}
// RemovalDue is the tool grants_removal_due: removal-due written again with a short time to live, so its marker starts
// the removal (the recovery for a marker missed while the part was down). Recorded as a hand act.
func (p *Part) RemovalDue(caller, why, ttl string) (any, error) {
if strings.TrimSpace(why) == "" {
return nil, errors.New("grants_removal_due needs a why, kept with the hand act")
}
d, err := time.ParseDuration(ttl)
if err != nil || d < time.Second || d > RemovalAfter {
return nil, fmt.Errorf("ttl %q is no time to live between 1s and %s", ttl, RemovalAfter)
}
r, err := p.readLease()
if err != nil {
return nil, err
}
if r.Handed.IsZero() || !r.Removed.IsZero() {
return nil, errors.New("no removal is due: the lease is not handed to mesh-grants, or the removal ran")
}
if err := p.putRemovalDue(d); err != nil {
return nil, err
}
p.recordHandAct("wrote removal-due again with "+d.String()+" to live", why+" (by "+caller+")")
return map[string]any{"written": KeyRemovalDue, "ttl": d.String(), "says": "its marker starts the removal when it expires"}, nil
}
// LeaseState answers where the lease write stands, for grants_lease_check.
func (p *Part) LeaseState() (LeaseRecord, error) { return p.readLease() }
modules/mesh-grants/internal/grants/memberships.go
+3 -16
View File
@@ -14,9 +14,8 @@ import (
"git.novox.be/novox/mesh-sdk/go/membership"
)
// Memberships are published for real from this milestone (the operator, 2026-10-25 22:31): since release 6 nobody
// publishes them, so a module placed since cannot serve its seat or reach a new state. The user list, the assertions and
// the grants stay in shadow until take-over.
// Memberships are published for real (the operator, 2026-10-25 22:31): since release 6 nobody else publishes them, so a
// module placed since could not serve its seat or reach a new state without them.
// MembershipGaps are what a membership carries that the part cannot compose from the bus.
var MembershipGaps = []Gap{
@@ -24,14 +23,6 @@ var MembershipGaps = []Gap{
{Input: "membership: root-free machines", Why: "asked of the controller's root-free verb at each pass and kept for that pass alone; a machine it does not call free (an error included) is composed as not free, and the pass says why"},
}
// Publisher says a membership the part wrote to its state memberships on its subject (membership.Subject), as a plain
// message, for a runtime that follows the subject alone. The state is where a membership is published and read back
// (ADR 0571 clause 7); the subject is a notice that depends on no stream, so the assignments stream's removal at the
// lease write (research 072 row 33) stops nothing. A refusal is said and the membership stays written.
type Publisher interface {
PublishMembership(node, module string, body []byte) error
}
// Membership is one module's membership on one node, as composed now, and what became of its publishing.
type Membership struct {
Node string `json:"node"`
@@ -76,11 +67,7 @@ const (
// its key in the part's own state memberships, found when it is there), carrying that one's private network and
// received contributions. It answers m as composed and published, or why not. held is every holding the memberships
// composed now carry (Holdings.ByMemberships). Pure: the caller writes and notifies.
func publish(p Publisher, was []byte, found bool, m Membership, h Holdings, held map[string]bool) (Membership, bool) {
if p == nil {
m.Published = "not published: no publisher here (only the part's process publishes)"
return m, false
}
func publish(was []byte, found bool, m Membership, h Holdings, held map[string]bool) (Membership, bool) {
if found {
// A value of the state is the bare membership, or the wrapper written before 2026-10-30 (mesh-sdk's one codec);
// one that is neither is no last membership, and m is a first one.
modules/mesh-grants/internal/grants/part.go
+15 -61
View File
@@ -9,7 +9,6 @@ import (
"sort"
"strings"
"sync"
"sync/atomic"
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
@@ -23,18 +22,16 @@ const (
StateUserList = "user-list"
StateMemberships = "memberships"
StateAssertions = "assertions"
StateTakeover = "takeover"
// StateGrants is the part's grants by module and fingerprint, the key layout of the controller's (grants.go).
StateGrants = delivery.PartGrantsState
KeyList = "list"
KeySwitch = "switch"
// KeyPass is the last pass's result, and KeyAsked a tool's ask to compose again now, both in StateUserList.
KeyPass = "pass"
KeyAsked = "asked"
)
// States is every state the part keeps.
var States = []string{StateAccounts, StateAssertions, StateGrants, StateLease, StateMemberships, StateTakeover, StateUserList, StateWithheld}
var States = []string{StateAccounts, StateAssertions, StateGrants, StateMemberships, StateUserList, StateWithheld}
// The events the part says.
const (
@@ -63,7 +60,7 @@ type Emitter interface {
Emit(event string, body any) error
}
// UserListState is the value of `user-list` in shadow: the list the grants part composed (users and grants, no hashes),
// UserListState is the value of `user-list`: the list the grants part composed (users and grants, no hashes),
// what the bus loaded, and how they differ. Inputs is the digest of all it says but At: a value is written again only
// when it changes.
type UserListState struct {
@@ -140,7 +137,7 @@ type Result struct {
}
// NothingTakesEffect is what every composition says it did on the bus beyond its own state and the memberships.
const NothingTakesEffect = "memberships (in its own state memberships, each said on its subject, unless one would leave a seat a placed module claims with no holder, or drop a capability its holder still asks for, which stays a shadow difference), and each bucket, log or stream a manifest declares that the bus lacks, created once with the manifest's limits (the operator, 2026-10-26 08:54); the user list, the grants and any change to a stream the bus holds stay in shadow until take-over"
const NothingTakesEffect = "memberships (in its own state memberships, unless one would leave a seat a placed module claims with no holder, or drop a capability its holder still asks for, which stays a shadow difference), each bucket, log or stream a manifest declares that the bus lacks, created once with the manifest's limits (the operator, 2026-10-26 08:54), the user list the bus node follows, the grants and the account consumers; a change to a stream the bus holds is only said"
// Part is the grants part: it reads, calls and writes through these, and nothing else.
type Part struct {
@@ -148,39 +145,21 @@ type Part struct {
Call Caller
Store Store
Emit Emitter
// Publish publishes memberships: the process's connection; nil in the tools, which ask the process instead.
Publish Publisher
// Create creates a declared stream the bus lacks: the process's connection; nil in the tools, which ask the process.
Create Creator
// Asker asks the operator (take-over, hand-back): the process's client of the router; nil in the tools, which write
// the request and let the process ask it.
Asker Asker
// Logf logs what no state or condition says (an error of acting on an answer): the process's; nil in the tools.
Logf func(string, ...any)
// Objects is the part's object bucket of sealed lists: the process's connection; nil in the tools.
Objects Objects
// Consumers creates an account's durable consumer from the lease write (consumers.go): the process's connection;
// nil in the tools, which compare only.
// Consumers creates an account's durable consumer (consumers.go): the process's connection; nil in the tools, which
// compare only.
Consumers ConsumerCreator
// Bus is where the nodes reach the bus, which each wanted account carries (accounts.go): the process's own credential's;
// empty in the tools, which write no account.
Bus BusAddress
// Say says a condition to the operator: the process's; nil in the tools.
Say func(asks.Condition) error
// Forget takes away the record the asker kept of an ask it failed to publish; Nonce is a test's id suffix.
Forget func(id string) error
Nonce func() string
// Lease is the process's hand on the controller's lease (leasewrite.go); nil in the tools, which ask the process.
Lease LeaseBus
// SaySeat says the seat mesh-grants' events (lease-handed, lease-altered, removal-failed): the process's; nil in the
// tools.
SaySeat SeatSayer
// deciding is set while a take-over is decided, or a lease act composes, so the composition it makes asks and
// reconciles nothing.
deciding atomic.Bool
// removalHeard is set when the marker of removal-due was heard, and taken by the next pass (leasewrite.go).
removalHeard atomic.Bool
Now func() time.Time
Now func() time.Time
cache Manifests
mu sync.Mutex
@@ -257,12 +236,10 @@ func (p *Part) Compose() Result {
// the trunk names, so the list grants a build's subjects before the want moves to it (issue 774).
recs, _ := Records(in)
listRecs, left := ListRecords(in)
// From the lease write's cut, the controller's write on the lease is taken from the list (ADR 0582 clause 3).
listRecs.LeaseCut = p.leaseCut()
holdings := HoldingsOf(in)
res.RootNotFree = WithRootFree(&listRecs, p.Call)
recs.RootFree = listRecs.RootFree
st := UserListState{V: 1, Mode: p.SwitchState(), LeftOut: left, Loaded: in.Loaded, BusNode: in.BusNode,
st := UserListState{V: 1, Mode: InForce, LeftOut: left, Loaded: in.Loaded, BusNode: in.BusNode,
Gaps: append(append([]Gap(nil), in.Gaps...), MembershipGaps...), At: p.now()}
if st.LeftOut == nil {
st.LeftOut = []Gap{}
@@ -299,7 +276,7 @@ func (p *Part) Compose() Result {
} else {
res.Revision, res.Written = rev, true
p.say(&res, EventUserList, map[string]any{"digest": st.Digest, "revision": rev, "users": st.Users,
"differing": len(st.Compare.Differing()), "mode": Shadow})
"differing": len(st.Compare.Differing()), "mode": InForce})
}
}
}
@@ -312,8 +289,8 @@ func (p *Part) Compose() Result {
p.writeWithheld(&res, in, recs)
}
list, n, leftOut, dropped, err := ComposeList(listRecs, in.Hashes, refused, kept)
// The grants the assigner reads once the lease is handed away, in shadow until then (ADR 0547 §4, §5); after take-over
// one that adds a permission only once the bus loaded a list that carries it (issue 774).
// The grants the assigner reads (ADR 0547 §4, §5): one that adds a permission only once the bus loaded a list that
// carries it (issue 774).
if p.Store != nil {
inForce, unknown := p.loadedContents(in, loaded, loadErr)
p.writeGrants(&res, in, listRecs, users, inForce, unknown)
@@ -322,7 +299,7 @@ func (p *Part) Compose() Result {
if err != nil {
res.Errors = append(res.Errors, "the user list could not be composed whole: "+err.Error())
} else {
// The users the bus loaded: what a first list after take-over must keep where the part has not yet written one.
// The users the bus loaded: what a first list must keep where the part has not yet written one.
var bus, lost []string
if st.Compare.Unread == "" {
bus = names(loaded)
@@ -340,17 +317,16 @@ func (p *Part) Compose() Result {
// The memberships are written to the state alone and never said on their subject, which nothing reads any more
// (novox/hq ADR 0571 clause 10, issue 774; unconditional since the controller's end, issue 1126).
for _, key := range sortedKeys(ms) {
// The last membership is the state's own (ADR 0571 clause 7), never the assignments stream's, which the bus
// module's tools remove at the lease write (research 072 row 33): a last read there would freeze every membership
// once it is gone.
// The last membership is the state's own (ADR 0571 clause 7), never the assignments stream's, which goes with the
// controller's leftovers (novox/hq ADR 0590 §6).
was, _, found, err := p.Store.Get(StateMemberships, key)
if err != nil {
res.NotPublished = append(res.NotPublished, key+": not published: the membership last published cannot be read from its own "+StateMemberships+": "+err.Error())
continue
}
m, now := publish(p.Publish, was, found, ms[key], holdings, seated)
m, now := publish(was, found, ms[key], holdings, seated)
// Published is written to the state, where every runtime reads it (process.FollowMembership); only one
// published, so a membership held back never reaches a reader. The subject is a notice after it.
// published, so a membership held back never reaches a reader.
if now {
body := membership.EncodeGrantsMembership(m.Membership)
if _, err := p.Store.Put(StateMemberships, key, body); err != nil {
@@ -406,31 +382,9 @@ func (p *Part) Compose() Result {
}
}
p.consumers(&res, recs, ask.Consumer)
if _, _, found, err := p.Store.Get(StateTakeover, KeySwitch); err == nil && !found {
data, _ := json.Marshal(Switch{V: 1, State: Shadow, Acts: []Act{}})
if _, err := p.Store.Put(StateTakeover, KeySwitch, data); err != nil {
res.Errors = append(res.Errors, fmt.Sprintf("its own %s cannot be written: %v", StateTakeover, err))
}
}
if !p.deciding.Load() {
if err := p.RaiseRequested(res.UserList); err != nil {
res.Errors = append(res.Errors, err.Error())
}
if err := p.Reconcile(); err != nil {
res.Errors = append(res.Errors, err.Error())
}
p.leaseStep(&res)
}
return res
}
// composeQuiet composes as a step of an act: asking and reconciling nothing.
func (p *Part) composeQuiet() Result {
p.deciding.Store(true)
defer p.deciding.Store(false)
return p.Compose()
}
// Last is the result of the last pass this part made, kept under the lock alone.
func (p *Part) Last() Result {
p.mu.Lock()
modules/mesh-grants/internal/grants/records.go
+1 -2
View File
@@ -36,8 +36,7 @@ func records(in Inputs, union bool) (buscompose.Records, []Gap) {
for _, own := range catalogue.SeatsWithAProtocol() {
seats[own.Name] = catalogue.SeatDeclaration{Name: own.Name, Scope: own.Scope, Accepts: own.Accepts, Emits: own.Emits, Serves: own.Serves}
}
// The enrolment users, from the seat mesh-enrolment's keys, and from the lease write through its redeem alone (ADR 0587
// §4, §6).
// The enrolment users, from the seat mesh-enrolment's keys, enrolling through its redeem alone (ADR 0587 §4, §6).
out := buscompose.Records{Assigned: map[string][]buscompose.Declared{}, People: map[string][]string{},
Interchangeable: map[string]bool{}, Enrolling: slices.Clone(in.Enrolling), LeaseHandedAway: true,
// And each node's logins (ADR 0583 §1): both while it rotates.
modules/mesh-grants/internal/grants/rootfree.go
+1 -1
View File
@@ -51,7 +51,7 @@ const EventAccountIssued = "mesh-vault." + buscompose.AccountIssuedEvent
// Consumes is every event the part hears, as its manifest's consumes name them.
//
// Not the controller's condition events: the keeper's alone (novox/hq ADR 0564 §9, issue 774).
var Consumes = []string{EventKeeperRaised, EventKeeperCleared, EventAccountIssued, EventBucketExpirySet}
var Consumes = []string{EventKeeperRaised, EventKeeperCleared, EventAccountIssued}
// IsAccountIssued says an event heard is the vault's account-issued, by its name or its bus subject.
func IsAccountIssued(event string) bool {
modules/mesh-grants/internal/grants/sealed.go
+16 -96
View File
@@ -14,13 +14,12 @@ import (
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// **The list the bus node loads from take-over** (novox/hq ADR 0547 clause 7, issue 1945). Each pass composes the bus's
// user list with the vault's hashes (the accounts block, buscompose.ComposeAccounts, as the bus module's file holds it),
// seals it to the bus node's key (the key the node says in its have of the bus module) and puts it as one object of the
// part's object bucket, named by its digest; then writes the key `user-list.sealed` naming it, with the revision, the user
// count, the users left out for want of a hash, and which list the bus node follows (the switch, clause 8). The object
// and the key are written in shadow too: the bus node reads the key and follows the controller's list until the key says
// FollowGrants, so a take-over has a list to load at once. Only the process writes them (Objects); a tool asks it.
// **The list the bus node loads** (novox/hq ADR 0547 clause 7, issue 1945). Each pass composes the bus's user list with
// the vault's hashes (the accounts block, buscompose.ComposeAccounts, as the bus module's file holds it), seals it to the
// bus node's key (the key the node says in its have of the bus module) and puts it as one object of the part's object
// bucket, named by its digest; then writes the key `user-list.sealed` naming it, with the revision, the user count and
// the users left out for want of a hash. The key always says FollowGrants: the bus node follows this list alone since the
// controller ended by its unassignment (ADR 0590, issue 1126). Only the process writes them (Objects); a tool asks it.
// KeySealed is the key of the state user-list the bus node reads (delivery.GrantsUserListKey).
const KeySealed = delivery.GrantsUserListKey
@@ -81,23 +80,14 @@ func (p *Part) Sealed() (delivery.GrantsUserList, uint64, bool) {
return g, rev, true
}
// followOf is which list the switch says the bus node follows.
func followOf(state string) string {
if state == TookOver {
return delivery.FollowGrants
}
return delivery.FollowController
}
// writeSealed puts the list sealed to the bus node and writes the key, each only when what it says changed. It says in
// res what it wrote, or why it wrote nothing.
func (p *Part) writeSealed(res *Result, list []byte, users int, leftOut, dropped []string, in Inputs, bus []string, lost []string, composed []User) {
cur, _, had := p.Sealed()
next := cur
next.Follow = followOf(p.SwitchState())
// The bus node follows this list alone (ADR 0571 clause 8): always since the controller ended by its unassignment,
// with no lease written (issue 1126).
next.LeaseHanded = true
// The bus node follows this list alone (ADR 0571 clause 8): always since the controller ended by its unassignment
// (novox/hq ADR 0590, issue 1126).
next.Follow, next.LeaseHanded = delivery.FollowGrants, true
key := ""
if in.Loaded != nil {
key = in.Loaded.Key
@@ -106,16 +96,16 @@ func (p *Part) writeSealed(res *Result, list []byte, users int, leftOut, dropped
why, held := "", ""
switch {
case p.Objects == nil:
// The tools: the process writes; the follow alone may change here, from a tool's view of the switch, and is not.
// The tools: the process writes.
res.Sealed = sealedSays(cur, had, "")
return
case in.BusNode == "":
why = "no node runs the bus's module, so the list is sealed to nobody"
case key == "":
why = "the bus node's have of the bus module names no key yet (its node-engine says it from this release)"
case next.Follow == delivery.FollowGrants && cur.Digest != digest && p.holdsBack(in, leftOut, dropped, bus, lost, digest) != "":
// After take-over the list in force is never replaced by one composed past a failed read, or one that drops a user
// of it for want of a hash: that user's module would be locked out of the bus (review of the switch).
case cur.Digest != digest && p.holdsBack(in, leftOut, dropped, bus, lost, digest) != "":
// The list in force is never replaced by one composed past a failed read, or one that drops a user of it for want
// of a hash: that user's module would be locked out of the bus (review of the switch).
why = p.holdsBack(in, leftOut, dropped, bus, lost, digest)
held = why
p.keepHashHold(res, digest)
@@ -198,7 +188,7 @@ func (p *Part) sayHeld(why string) {
}
}
// HeldCondition is the key of the warning raised while a list in force is held back after take-over.
// HeldCondition is the key of the warning raised while a list in force is held back.
const HeldCondition = "mesh-grants.user-list-held"
func sameSealed(a, b delivery.GrantsUserList) bool {
@@ -231,48 +221,6 @@ func sealedSays(g delivery.GrantsUserList, had bool, why string) *SealedState {
return s
}
// syncFollow writes the switch into the key at once, so the bus node turns on the operator's yes, not at the next pass.
func (p *Part) syncFollow(state string) error {
cur, _, had := p.Sealed()
if had && cur.Follow == followOf(state) {
return nil
}
if !had {
cur = delivery.GrantsUserList{}
}
cur.Follow, cur.At = followOf(state), p.now()
data, err := delivery.EncodeGrantsUserList(cur)
if err != nil {
return err
}
_, err = p.Store.Put(StateUserList, KeySealed, data)
return err
}
// Effect is where the switch stands on the bus node: "switched" only once the bus node's loaded list is the grants part's
// written one (its source the part's key, its digest the written digest); otherwise what it waits for.
func (p *Part) Effect(state string, loaded *delivery.UserListWritten) string {
g, _, had := p.Sealed()
fromGrants := loaded != nil && loaded.Source == delivery.GrantsUserListSource
switch {
case state == TookOver && had && fromGrants && g.Digest != "" && loaded.Digest == g.Digest && loaded.Error == "":
return "switched: the bus node loaded the grants part's list " + short(g.Digest) + " (revision " + fmt.Sprint(g.Revision) + ")"
case state == TookOver && !had || state == TookOver && g.Digest == "":
return "recorded, not in effect: the grants part has written no sealed list for the bus node yet"
case state == TookOver && loaded == nil:
return "recorded, not in effect: the bus module's have names no loaded list"
case state == TookOver && loaded.Unwatched != "" && !fromGrants:
return "recorded, not in effect: the bus node does not watch the grants part's key: " + loaded.Unwatched
case state == TookOver && loaded.Error != "":
return "recorded, not in effect: the bus node could not load the grants part's list: " + loaded.Error
case state == TookOver:
return "recorded, not in effect yet: the bus node loaded " + short(loaded.Digest) + " from " + loaded.Source + ", the grants part wrote " + short(g.Digest)
case fromGrants:
return "handed back, not in effect yet: the bus node still follows the grants part's list"
}
return "shadow: the bus node follows the controller's list"
}
func short(d string) string {
d = strings.TrimPrefix(d, "sha256:")
if len(d) > 12 {
@@ -281,36 +229,8 @@ func short(d string) string {
return d
}
// followReasons is what a take-over waits for on the sealed list itself: a list written for the bus node, of the list
// composed now, and the loaded list granting the bus node its read (the controller's composer carries ADR 0547 clause 7's
// grants from the operator's second exception; until its delivery the bus node could not read the first list).
func (p *Part) followReasons(res Result, in Inputs) []string {
var out []string
g, _, had := p.Sealed()
switch {
case !had || g.Digest == "":
why := "no list is written"
if res.Sealed != nil && res.Sealed.Why != "" {
why = res.Sealed.Why
}
out = append(out, "the grants part has written no sealed list for the bus node: "+why)
case in.Loaded == nil || in.Loaded.Key == "" || g.SealedTo != delivery.SealingKeyID(in.Loaded.Key):
out = append(out, "the sealed list is not sealed to the key the bus node says now")
}
if in.BusNode != "" {
node := "node." + in.BusNode
for _, d := range res.UserList.Compare.Differ {
if d.User == node && slices.ContainsFunc(d.PublishAdded, func(s string) bool { return strings.Contains(s, delivery.GrantsUserListBucket) }) {
out = append(out, "the loaded list does not grant "+node+" reading the grants part's sealed list (ADR 0547 clause 7): "+
"the controller's composer carries it from its delivery of the operator's second exception")
}
}
}
return out
}
// KeyWrittenUsers is the key of the state user-list naming the users of the list the key `sealed` names: what a list
// composed after take-over must keep.
// composed must keep.
const KeyWrittenUsers = "sealed-users"
// KeyWrittenCredentials is the key of the state user-list naming, per user of the list in force, which password its hash
@@ -371,7 +291,7 @@ func listUsers(list []byte) []string {
return out
}
// holdsBack says why a list composed now may not replace the one in force after take-over: a read failed this pass, a
// holdsBack says why a list composed now may not replace the one in force: a read failed this pass, a
// user of the list in force is left out for want of a hash, or the list would leave a seat the list in force holds with
// no holder while a placed module still claims it (lost, Holdings.LastHolders: the membership's guard, issue 1945; the
// loaded list not read holds it too, since that cannot be ruled out). The list in force is the one the part wrote, by the users
modules/mesh-grants/internal/grants/takeover.go
+5 -552
View File
@@ -1,556 +1,9 @@
package grants
import (
"git.novox.be/novox/mesh-sdk/go/handacts"
"crypto/rand"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/asker"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The switch between the controller's user list and the grants part's (ADR 0547 clause 8). The operator's word is a
// warrant: the operator's answer to an ask on a channel that proves who answered, never a caller's name, since an
// operator's session and every agent's call as the same machine account (the operator, 2026-10-26).
// Shadow is the switch's state until take-over: the bus node follows the controller's list. TookOver is its state after.
const (
Shadow = "shadow"
TookOver = "took-over"
)
// KeyRequest is the key in StateTakeover where a tool leaves its request for the process to ask.
const KeyRequest = "request"
// The two acts the operator may warrant.
const (
ActTakeOver = "take-over"
ActHandBack = "hand-back"
)
// Switch is the value of `takeover`: which list the bus node follows, every act on the switch and the ask open now.
type Switch struct {
V int `json:"v"`
State string `json:"state"`
// Open is the id of the ask the operator has not answered yet, empty when none.
Open string `json:"open,omitempty"`
// Refusing are the acts whose refusal condition was said and not cleared yet.
Refusing []string `json:"refusing,omitempty"`
Acts []Act `json:"acts"`
}
// Act is one take-over or hand-back asked, answered, done or refused.
type Act struct {
Act string `json:"act"`
By string `json:"by"`
Why string `json:"why,omitempty"`
At time.Time `json:"at"`
// Ask is the router's ask of the act; Said what became of it: asked, switched, declined, expired, refused.
Ask string `json:"ask,omitempty"`
Said string `json:"said,omitempty"`
Refused string `json:"refused,omitempty"`
}
// Request is what a tool leaves for the process to ask: written once per act, marked raised when asked.
type Request struct {
ID string `json:"id"`
Act string `json:"act"`
By string `json:"by"`
Why string `json:"why,omitempty"`
At time.Time `json:"at"`
Raised bool `json:"raised,omitempty"`
// RaisedAt is when it was asked, and Failed why asking it failed (the ask is then closed as refused).
RaisedAt time.Time `json:"raised_at,omitempty"`
Failed string `json:"failed,omitempty"`
// Adds is the digest of the rights the grants part's list added when the take-over was asked (Comparison.AddedDigest):
// the rights the ask named, which its yes answers (ADR 0562 §3).
Adds string `json:"adds,omitempty"`
}
// Asker is the client of the router the part asks through (mesh-sdk asker.Client).
type Asker interface {
Ask(a asks.Ask) error
// Where is the router's record of an ask: its state and its warrant, if any.
Where(id string) (string, *asks.Warrant, error)
// Take hands a warrant to act once (a warrant the process missed is caught up through it).
Take(w asks.Warrant, act func(asks.Option, asks.Warrant) error) (asker.Handled, error)
// Cancel takes an ask back at the router, so a copy the operator may still see ends there too.
Cancel(id string) error
}
// askMargin is how long past an ask's expiry the router is given to say it ended, before the part closes it itself;
// recordGrace how long a just-asked ask may be missing from the router's record.
const (
askMargin = 5 * time.Minute
recordGrace = time.Minute
)
// keptActs is how many acts the switch keeps, newest last.
const keptActs = 50
func (p *Part) readSwitch() (Switch, error) {
s := Switch{V: 1, State: Shadow}
raw, _, found, err := p.Store.Get(StateTakeover, KeySwitch)
if err != nil {
return s, err
}
if found {
_ = json.Unmarshal(raw, &s)
}
return s, nil
}
func (p *Part) writeSwitch(s Switch) error {
if len(s.Acts) > keptActs {
s.Acts = s.Acts[len(s.Acts)-keptActs:]
}
if s.Acts == nil {
s.Acts = []Act{}
}
data, _ := json.Marshal(s)
_, err := p.Store.Put(StateTakeover, KeySwitch, data)
return err
}
// SwitchState is which list the bus node follows now.
func (p *Part) SwitchState() string {
if s, err := p.readSwitch(); err == nil && s.State != "" {
return s.State
}
return Shadow
}
// TakeOver is the tool: it asks the operator to switch the bus node to the grants part's list. It writes the request,
// and the part's process asks it on the operator's channel; only the operator's yes (a warrant) runs the switch, after
// the refusals of Decide. Asking again while an ask is open is refused.
func (p *Part) TakeOver(caller, why string) (any, error) { return p.request(ActTakeOver, caller, why) }
// HandBack is the tool: it is answered as gone. The controller ended by its unassignment (issue 1126), so nothing
// writes its list and the grants part's list is the only one (ADR 0547 §11).
func (p *Part) HandBack(caller, why string) (any, error) {
return nil, errors.New("gone: the controller ended by its unassignment (issue 1126), so nothing writes its user list; " +
"the grants part's list is the only one and the user list is not handed back (ADR 0547 §11)")
}
// nonce is four hex digits that keep two asks of one second apart.
func (p *Part) nonce() string {
if p.Nonce != nil {
return p.Nonce()
}
var b [2]byte
_, _ = rand.Read(b[:])
return hex.EncodeToString(b[:])
}
func (p *Part) request(act, caller, why string) (any, error) {
s, err := p.readSwitch()
if err != nil {
return nil, fmt.Errorf("the switch cannot be read: %w", err)
}
a := Act{Act: act, By: caller, Why: why, At: p.now()}
if s.Open != "" {
return nil, fmt.Errorf("%s is already asked: the ask %s waits for the operator's answer", act, s.Open)
}
if act == ActHandBack && s.State == Shadow {
a.Refused = "nothing to hand back: the grants part is in shadow and never took over, so the bus node follows the controller's list already"
s.Acts = append(s.Acts, a)
_ = p.writeSwitch(s)
return nil, fmt.Errorf("%s", a.Refused)
}
if act == ActTakeOver && s.State == TookOver {
return nil, fmt.Errorf("the bus node follows the grants part's list already")
}
a.Ask = act + "-" + p.now().Format("20060102t150405") + "-" + p.nonce()
a.Said = "asked"
req, _ := json.Marshal(Request{ID: a.Ask, Act: act, By: caller, Why: why, At: a.At})
if _, err := p.Store.Put(StateTakeover, KeyRequest, req); err != nil {
return nil, fmt.Errorf("the request could not be written: %w", err)
}
s.Open = a.Ask
s.Acts = append(s.Acts, a)
if err := p.writeSwitch(s); err != nil {
return nil, fmt.Errorf("the act could not be recorded: %w", err)
}
// The process asks at its next pass: wake it.
_, _ = p.AskCompose(caller)
return map[string]any{"asked": a.Ask, "says": "the part's process asks the operator on their channel; only the operator's yes records the switch, and it takes effect when the bus node loads the grants part's list. grants_take_over_state shows where it stands"}, nil
}
// RaiseRequested is the process's: it asks the operator the request a tool left, once, naming the rights the pass's
// comparison c adds. Where the part has no asker (the tools) it does nothing.
func (p *Part) RaiseRequested(st UserListState) error {
if p.Asker == nil {
return nil
}
raw, rev, found, err := p.Store.Get(StateTakeover, KeyRequest)
if err != nil || !found {
if err != nil {
return fmt.Errorf("its own %s %s cannot be read: %v", StateTakeover, KeyRequest, err)
}
return nil
}
var r Request
if json.Unmarshal(raw, &r) != nil || r.Raised || r.Failed != "" || r.ID == "" {
return nil
}
if s, err := p.readSwitch(); err != nil || s.Open != r.ID {
return nil // an ask no longer open is not asked
}
ask, err := p.AskOf(r, st)
if err != nil {
return err
}
if r.Act == ActTakeOver {
r.Adds = st.Compare.AddedDigest()
}
if err := p.Asker.Ask(ask); err != nil {
// Closed as refused, and the half-made record the asker kept before publishing is taken away, so nothing stays open.
r.Failed = err.Error()
data, _ := json.Marshal(r)
_, _ = p.Store.Put(StateTakeover, KeyRequest, data)
if p.Forget != nil {
_ = p.Forget(r.ID)
}
// A publish that timed out may still have reached the router: taken back there, so a yes to it cannot land on an
// ask the part has forgotten.
if cerr := p.Asker.Cancel(r.ID); cerr != nil && p.Logf != nil {
p.Logf("the ask %s that could not be published was not taken back at the router: %v", r.ID, cerr)
}
if s, rerr := p.readSwitch(); rerr == nil {
_ = p.finish(s, Act{Act: r.Act, By: "operator", At: p.now(), Ask: r.ID, Said: "refused", Refused: "the operator could not be asked: " + err.Error()})
}
return fmt.Errorf("the operator could not be asked the %s: %v", r.Act, err)
}
r.Raised, r.RaisedAt = true, p.now()
data, _ := json.Marshal(r)
if _, err := p.Store.Put(StateTakeover, KeyRequest, data); err != nil {
return fmt.Errorf("the ask %s was asked and its request could not be marked: %v", r.ID, err)
}
_ = rev
return nil
}
// Reconcile closes an ask that can stay open for ever, once per pass while one is open: a warrant the process missed is
// caught up from the router's record, one that ended with none is closed, and one past its expiry and a margin is closed
// as expired by the part itself.
func (p *Part) Reconcile() error {
if p.Asker == nil {
return nil
}
s, err := p.readSwitch()
if err != nil || s.Open == "" {
return err
}
act := actOf(s.Open)
at := p.now()
for _, a := range s.Acts {
if a.Ask == s.Open {
at = a.At
break
}
}
state, w, err := p.Asker.Where(s.Open)
if err != nil {
return fmt.Errorf("the router's record of the ask %s cannot be read: %v", s.Open, err)
}
if w != nil {
h, herr := p.Asker.Take(*w, p.Warranted)
if h == "" && herr != nil {
// The part's own record of the ask could not be reached: nothing was taken, and the next pass tries again.
return fmt.Errorf("the operator's answer to %s could not be taken yet: %v", s.Open, herr)
}
p.Said(h, *w, herr)
if cur, rerr := p.readSwitch(); rerr == nil && cur.Open == s.Open {
_ = p.finish(cur, Act{Act: act, By: "operator", At: p.now(), Ask: s.Open, Said: string(w.Outcome)})
}
return nil
}
raw, _, found, _ := p.Store.Get(StateTakeover, KeyRequest)
var r Request
if found {
_ = json.Unmarshal(raw, &r)
}
switch {
case p.now().After(at.Add(asks.ApproveLasts + askMargin)):
return p.finish(s, Act{Act: act, By: "operator", At: p.now(), Ask: s.Open, Said: string(asks.OutcomeExpired)})
case state == "" && r.ID == s.Open && r.Raised && p.now().After(r.RaisedAt.Add(recordGrace)):
return p.finish(s, Act{Act: act, By: "operator", At: p.now(), Ask: s.Open, Said: "ended", Refused: "the router holds no record of the ask"})
}
return nil
}
func actOf(id string) string {
if strings.HasPrefix(id, ActHandBack+"-") {
return ActHandBack
}
return ActTakeOver
}
// Following says whether the bus node has loaded the grants part's list: "yes" when its loaded digest equals the part's,
// "not yet" when it loaded another, "unknown" when its have says none.
func (p *Part) Following() string {
loaded := ReadInputs(p.Read, &p.cache).Loaded
g, _, had := p.Sealed()
if loaded == nil || !had || g.Digest == "" {
return "unknown: the bus module's have names no loaded list, or the grants part wrote none"
}
if loaded.Source == delivery.GrantsUserListSource && loaded.Digest == g.Digest {
return "yes"
}
return "not yet: the bus node loaded another list"
}
// AskOf is the ask of a request, in the words the operator reads: what changes, why, what it risks; for a take-over every
// right the grants part's list adds by the user list st compares now, by user, or what a yes would be refused for
// (ADR 0562 §3, TakeOverWords). Its yes binds the act.
func (p *Part) AskOf(r Request, st UserListState) (asks.Ask, error) {
c := st.Compare
verb, headline, does, stays, expl := "grants_take_over", "Switch the bus's user list to the grants part?", "The switch is recorded; it takes effect when the bus node loads the grants part's list.",
"The controller's list stays in force.", "The grants part has composed the user list the bus holds today. Your yes records the switch; it takes effect when the bus node loads the part's list.\n"+
"If the lists differ other than by an added right, or a user has no hash in the vault, the switch is refused and you are told which."
if r.Act == ActHandBack {
verb, headline, does, stays, expl = "grants_hand_back", "Give the bus's user list back to the controller?", "The switch back is recorded; it takes effect when the bus node loads the controller's list.",
"The grants part's list stays in force.", "The bus follows the grants part's list now.\nThe controller still writes its list until it loses its lease, so the bus can follow it again."
}
details := "Asked by " + r.By + ": " + r.Why
act := asks.Act{"verb": verb, "module": "mesh-grants", "ask": r.ID}
if summary, points, list := TakeOverWords(c, st.Credentials); r.Act == ActTakeOver && summary != "" {
if len(Refusals(c, st.Credentials)) > 0 {
expl = summary + "\n" + strings.Join(points, "\n")
details = "Refused for:\n" + list + "\n" + details
} else {
expl = summary + " Your yes records the switch; it takes effect when the bus node loads the part's list.\n" + strings.Join(points, "\n")
details = "Added, by user:\n" + list + "\nA user added or dropped, a right taken away or a missing hash refuses the switch, naming them.\n" + details
}
act["adds"] = c.AddedDigest()
}
bind, err := asks.ActDigest(act)
if err != nil {
return asks.Ask{}, err
}
return asks.Ask{ID: r.ID, Headline: headline, Explanation: expl, Who: asks.Operator,
Options: []asks.Option{{ID: "yes", Label: "Yes, switch", Does: does, Level: asks.Approve, Binds: bind},
{ID: "no", Label: "No, leave it", Does: stays, Level: asks.Acknowledge}},
Expires: p.now().Add(asks.ApproveLasts - time.Minute), OnExpiry: stays + " Nothing changes.", About: "mesh-grants-" + r.Act,
Details: details}, nil
}
// Warranted is the process's act on the operator's answer (asker.Client.Listen hands it once, after checking the ask, the
// option and its level). A no is recorded; a yes runs the switch after the refusals, and a refusal is said to the operator.
func (p *Part) Warranted(o asks.Option, w asks.Warrant) error {
if isLeaseAsk(w.Ask) {
return p.leaseWarranted(o, w)
}
s, err := p.readSwitch()
if err != nil {
return err
}
act := ActTakeOver
if strings.HasPrefix(w.Ask, ActHandBack+"-") {
act = ActHandBack
}
if s.Open != w.Ask {
return fmt.Errorf("the warrant answers the ask %s, and the ask open is %q: nothing is switched", w.Ask, s.Open)
}
a := Act{Act: act, By: "operator", At: p.now(), Ask: w.Ask}
if o.ID != "yes" {
a.Said = "declined"
return p.finish(s, a)
}
var reasons []string
if act == ActTakeOver {
reasons = p.decide(w.Ask)
if s, err = p.readSwitch(); err != nil { // Decide clears a refusal and writes the switch: read it again
return err
}
} else if s.State == Shadow {
reasons = []string{"nothing to hand back: the bus node follows the controller's list already"}
}
if len(reasons) > 0 {
a.Said, a.Refused = "refused", strings.Join(reasons, "; ")
p.sayRefused(&s, act, a.Refused)
return p.finish(s, a)
}
p.clearRefused(&s)
a.Said = "recorded"
if act == ActTakeOver {
s.State = TookOver
} else {
s.State = Shadow
}
if err := p.finish(s, a); err != nil {
return err
}
p.recordWarrant(w.Ask)
// The bus node turns on the key: written now, so the switch reaches it on the operator's yes (ADR 0547 clause 8).
if err := p.syncFollow(s.State); err != nil {
return fmt.Errorf("the switch is recorded and the key the bus node follows could not be written (the next pass writes it): %w", err)
}
if p.Emit != nil {
_ = p.Emit.Emit(EventUserList, map[string]any{"switch": s.State})
}
return nil
}
// Said is the process's note of an ask that ended with no yes or no (expired, or refused by the router): the switch
// stays, and the ask is closed so the operator can be asked again.
func (p *Part) Said(h asker.Handled, w asks.Warrant, err error) {
if err != nil && p.Logf != nil {
p.Logf("the answer to %s (%s) ended in an error: %v", w.Ask, h, err)
}
if isLeaseAsk(w.Ask) {
if r, err := p.readLease(); err == nil && r.Open != nil && r.Open.Ask == w.Ask && r.Open.Step == StepAsked {
r.end(string(w.Outcome), string(w.Outcome)+" ("+string(h)+"): nothing was written", p.now())
_ = p.writeLease(r)
}
return
}
s, err := p.readSwitch()
if err != nil || s.Open != w.Ask {
return
}
_ = p.finish(s, Act{Act: actOf(w.Ask), By: "operator", At: p.now(), Ask: w.Ask, Said: string(w.Outcome) + " (" + string(h) + ")"})
}
func (p *Part) finish(s Switch, a Act) error {
if s.Open == a.Ask {
s.Open = ""
}
s.Acts = append(s.Acts, a)
return p.writeSwitch(s)
}
// Decide is what stands between a take-over and the grants part's list, read from a composition made now: the loaded list
// not read, any user the two lists keep apart other than by a right the shadow list adds (a user added, a loaded user
// dropped that placement still names, a right a loaded user would lose: ADR 0562, which changes ADR 0547 clause 8's
// equality), and any user with no hash in the vault.
// Nothing here but what is true now: a take-over is refused for these and for no other reason. Empty means none.
func (p *Part) Decide() []string {
return p.decide("")
}
// decide is Decide for the yes to the ask (none: empty). A right the shadow list adds is no reason (ADR 0562 §1): the ask
// names it, and the yes covers the rights the ask named and no other (ADR 0562 §3).
func (p *Part) decide(ask string) []string {
p.deciding.Store(true)
res := p.Compose()
p.deciding.Store(false)
reasons := Refusals(res.UserList.Compare, res.UserList.Credentials)
if ask != "" && p.askedAdds(ask) != res.UserList.Compare.AddedDigest() {
reasons = append(reasons, "the rights the grants part's list adds changed since you were asked, so the yes does not cover them: ask the take-over again")
}
reasons = append(reasons, p.followReasons(res, ReadInputs(p.Read, &p.cache))...)
if len(reasons) == 0 {
if s, err := p.readSwitch(); err == nil && len(s.Refusing) > 0 {
p.clearRefused(&s)
_ = p.writeSwitch(s)
}
}
return reasons
}
// askedAdds is the digest of the added rights the ask named: empty when its request is not the one kept.
func (p *Part) askedAdds(ask string) string {
raw, _, found, err := p.Store.Get(StateTakeover, KeyRequest)
if err != nil || !found {
return ""
}
var r Request
if json.Unmarshal(raw, &r) != nil || r.ID != ask {
return ""
}
return r.Adds
}
func contains(s []string, v string) bool {
for _, x := range s {
if x == v {
return true
}
}
return false
}
// sayRefused tells the operator a yes was given and the switch refused, with the reasons (the users that differ named). The
// condition stays raised until a switch is recorded or a later Decide finds no reason (clearRefused).
func (p *Part) sayRefused(s *Switch, act, why string) {
if p.Say == nil {
return
}
c := asks.Condition{Key: "mesh-grants." + act + "-refused", Severity: asks.SeverityWarning, Subject: asks.Scope{Scope: "module", ID: "mesh-grants"},
State: asks.StateRaised, Summary: act + " refused: " + why, Headline: "The user list switch was refused", Needs: "a look at why",
Explanation: asks.NeedsYou + "a look at why. You said yes to the switch and it was refused: " + why, Resolved: "the switch is asked again and goes through"}
if p.Say(c) == nil && !contains(s.Refusing, act) {
s.Refusing = append(s.Refusing, act)
}
}
// clearRefused clears every refusal condition said and not cleared: on a recorded switch, a hand-back, or a Decide with no reason.
func (p *Part) clearRefused(s *Switch) {
if p.Say == nil {
return
}
var left []string
for _, act := range s.Refusing {
c := asks.Condition{Key: "mesh-grants." + act + "-refused", Subject: asks.Scope{Scope: "module", ID: "mesh-grants"}, State: asks.StateCleared}
if p.Say(c) != nil {
left = append(left, act)
}
}
s.Refusing = left
}
// TakeOverState is the tool take-over-state: the switch, the request, and what became of each ask.
func (p *Part) TakeOverState() (any, error) {
s, err := p.readSwitch()
if err != nil {
return nil, err
}
loaded := ReadInputs(p.Read, &p.cache).Loaded
out := map[string]any{"state": s.State, "open": s.Open, "acts": s.Acts, "bus_node_loaded_grants_list": p.Following(),
"effect": p.Effect(s.State, loaded),
"says": "a recorded switch takes effect when the bus node loads the grants part's list; effect says switched only once the bus node's loaded digest is the grants part's written one"}
if g, rev, had := p.Sealed(); had {
out["written"] = map[string]any{"digest": g.Digest, "revision": g.Revision, "key_revision": rev, "follow": g.Follow,
"users": g.Users, "left_out": g.LeftOut, "sealed_to": g.Node}
}
if loaded != nil {
out["loaded"] = map[string]any{"source": loaded.Source, "digest": loaded.Digest, "revision": loaded.Revision,
"users": loaded.Users, "follows": loaded.Follows, "at": loaded.At, "error": loaded.Error}
}
if raw, _, found, err := p.Store.Get(StateTakeover, KeyRequest); err == nil && found {
out["request"] = json.RawMessage(raw)
}
return out, nil
}
// Module is the grants part's module, the asker of its take-over and hand-back.
// Module is the grants part's module.
const Module = "mesh-grants"
// The hand-act log's verb the part records its switches by (novox/hq ADR 0565 §11, ADR 0547 §6): the seat's.
const CallRecordWarrant = "seat:" + handacts.Seat + ".warranted"
// recordWarrant records, in the hand-act log, that a take-over or a hand-back was made on the operator's warrant for ask.
// Said, never fatal: the switch stands either way.
func (p *Part) recordWarrant(ask string) {
if p.Call == nil {
return
}
rec := handacts.Recorder{Ask: p.Call.Call}
taken, err := rec.RecordWarrant(Module, ask)
if p.Logf == nil {
return
}
if err != nil {
p.Logf("the switch on the warrant %s is not recorded in the hand-act log: %v", ask, err)
return
}
p.Logf("the switch on the warrant %s is recorded in the log of %s", ask, taken.Log)
}
// InForce is the mode of the part's user list: the bus node follows it alone (novox/hq ADR 0547 clause 8). The
// operator's take-over recorded it on 2026-10-27, and the controller's end by its unassignment made it the only list
// (ADR 0590 §3, issue 1126): no take-over and no hand-back are asked any more.
const InForce = "took-over"
modules/mesh-grants/internal/grants/tools.go
+6 -19
View File
@@ -7,33 +7,26 @@ import (
"slices"
"strings"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/membership"
)
// The part's tools, as its manifest lists them. Module tools in this milestone: a `mesh-*` seat must be in mesh-sdk's own
// seat list, which the running controller's composer does not know, so the part claims no seat until take-over.
// The part's tools, as its manifest lists them. The part claims no seat: the seat mesh-grants served the lease write
// alone, which went with the controller (novox/hq ADR 0590 §3, issue 1126).
const (
ToolGrants = "grants_grants"
ToolMembership = "grants_membership"
ToolUserList = "grants_user_list"
ToolComposeNow = "grants_compose_now"
ToolAssert = "grants_assert"
ToolTakeOver = "grants_take_over"
ToolTakeOverState = "grants_take_over_state"
ToolHandBack = "grants_hand_back"
ToolAccounts = "grants_accounts"
ToolConsumerMatch = "grants_consumer_match"
ToolAcceptHashes = "grants_accept_hashes"
)
// Tools is every tool, in the manifest's order.
var Tools = []string{ToolGrants, ToolMembership, ToolUserList, ToolComposeNow, ToolAssert, ToolTakeOver, ToolHandBack, ToolTakeOverState,
ToolAccounts, ToolConsumerMatch, ToolAcceptHashes, ToolLeaseCheck, ToolRemovalDue}
// SeatTool is the tool serving one of the seat mesh-grants' verbs, `mesh-grants.<verb>`.
func SeatTool(verb string) string { return catalogue.GrantsSeat + "." + verb }
var Tools = []string{ToolGrants, ToolMembership, ToolUserList, ToolComposeNow, ToolAssert, ToolAccounts, ToolConsumerMatch,
ToolAcceptHashes}
// WritesGrants is what grants says of the part's own grants: written once the list the bus loaded carries a build's
// accounts, and the one key the assigner reads (ADR 0571 clause 9, which changes ADR 0547 clause 5).
@@ -210,11 +203,6 @@ func (p *Part) Pass() Result {
return res
}
// NoGrantAgain is what compose-now says of the controller's grants: the controller serves no verb that writes a module's
// grant again (its issue replaces the account, which is not that), so a grant write it lost stays lost until it composes
// again; the part writes grants only from ADR 0547's third milestone.
const NoGrantAgain = "missing: a verb to ask the controller to write a module's grant again (mesh-controller serves none; its issue replaces the bus account, which is not that), so compose-now cannot ask it"
// AskCompose is the tool compose-now: it asks the part's process to compose and publish now (only the process holds the
// grant to publish a membership), and answers the last pass, whose revision moves when the process answered.
func (p *Part) AskCompose(caller string) (any, error) {
@@ -223,8 +211,7 @@ func (p *Part) AskCompose(caller string) (any, error) {
if err != nil {
return nil, fmt.Errorf("the ask could not be written: %w", err)
}
out := map[string]any{"asked": rev, "says": "the part's process composes and publishes at this ask; grants_user_list's last pass says what it published",
"controller_grants": NoGrantAgain}
out := map[string]any{"asked": rev, "says": "the part's process composes and publishes at this ask; grants_user_list's last pass says what it published"}
if raw, prev, found, err := p.Store.Get(StateUserList, KeyPass); err == nil && found {
out["last_pass"] = map[string]any{"revision": prev, "result": json.RawMessage(raw)}
}
@@ -233,7 +220,7 @@ func (p *Part) AskCompose(caller string) (any, error) {
// UserList answers the tool user-list: the shadow list as last written, beside what the bus node says it loaded now.
func (p *Part) UserList() (any, error) {
out := map[string]any{"mode": Shadow}
out := map[string]any{"mode": InForce}
raw, rev, found, err := p.Store.Get(StateUserList, KeyList)
switch {
case err != nil:
modules/mesh-grants/internal/grants/userlist.go
+1 -220
View File
@@ -13,7 +13,6 @@ import (
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/outward"
)
// User is one bus user and its grants, as the bus module's nats_users answers them: never a password hash.
@@ -317,7 +316,7 @@ func (c Comparison) Removing() []string {
// AddedRights are the rights the shadow list adds, by user: every user's added publish and subscribe rights and an
// allowance of responses gained, whatever else the comparison holds. They are the whole difference only where Removing is
// empty, and only then is a take-over allowed for them (ADR 0562 §1, §2).
// empty (ADR 0562 §1, §2).
func (c Comparison) AddedRights() []UserDiff {
var out []UserDiff
for _, d := range c.Differ {
@@ -332,224 +331,6 @@ func (c Comparison) AddedRights() []UserDiff {
return out
}
// AddedDigest is the digest of every right the shadow list adds: what an ask names and its yes answers (ADR 0562 §3).
func (c Comparison) AddedDigest() string {
var lines []string
for _, d := range c.AddedRights() {
for _, s := range d.PublishAdded {
lines = append(lines, d.User+"\tpublish\t"+s)
}
for _, s := range d.SubscribeAdded {
lines = append(lines, d.User+"\tsubscribe\t"+s)
}
if d.Responses != "" {
lines = append(lines, d.User+"\tresponses")
}
}
sort.Strings(lines)
sum := sha256.Sum256([]byte(strings.Join(lines, "\n")))
return "sha256:" + hex.EncodeToString(sum[:])
}
// The kinds an added right is told by in the operator's ask.
const (
kindAdvisory = "advisories"
kindRead = "reads"
kindCall = "calls"
kindEvent = "events"
kindOther = "other"
)
var rightKinds = []string{kindRead, kindAdvisory, kindCall, kindEvent, kindOther}
func kindOfRight(subject string) string {
switch {
case strings.HasPrefix(subject, "$JS.EVENT.ADVISORY."):
return kindAdvisory
case strings.HasPrefix(subject, "$JS.") || strings.HasPrefix(subject, "$KV.") || strings.HasPrefix(subject, "$O."):
return kindRead
case strings.HasPrefix(subject, "mesh.seat."):
return kindCall
case strings.HasPrefix(subject, "mesh.mod.") && strings.Contains(subject, ".event."):
return kindEvent
}
return kindOther
}
// kindWords says what each kind is, in the operator's words.
var kindWords = map[string]string{kindRead: "reads of buckets and streams", kindAdvisory: "stream-created advisories",
kindCall: "seat calls", kindEvent: "events", kindOther: "other subjects"}
// tally is one user's added rights counted by kind, as "36 reads, 1 call"; responses allowed are said too.
func tally(d UserDiff) string {
n := map[string]int{}
for _, s := range append(append([]string(nil), d.PublishAdded...), d.SubscribeAdded...) {
n[kindOfRight(s)]++
}
var parts []string
for _, k := range rightKinds {
if n[k] > 0 {
word := k
if n[k] == 1 {
word = map[string]string{kindAdvisory: "advisory", kindRead: "read", kindCall: "call", kindEvent: "event", kindOther: "other"}[k]
}
parts = append(parts, fmt.Sprintf("%d %s", n[k], word))
}
}
if d.Responses != "" {
parts = append(parts, "responses allowed")
}
return strings.Join(parts, ", ")
}
// AddedWordsBudget is how many characters the ask's list of added rights may take, so the ask fits one message on a
// channel such as Telegram (4096 characters) with its headline, explanation and options.
const AddedWordsBudget = 2200
// AddedWords are an ask's words for the rights the shadow list adds (ADR 0562 §3): a summary, a point per kind with its
// total, and the list by user, users with the same count grouped on one line; within the content rule (outward.Check,
// a name that would not pass written with spaces) and AddedWordsBudget, past which the last lines are counted, not named.
func (c Comparison) AddedWords() (summary string, points []string, list string) {
adds := c.AddedRights()
if len(adds) == 0 || c.Unread != "" || len(c.Removing()) > 0 {
return "", nil, ""
}
total, byKind, usersOf := 0, map[string]int{}, map[string]int{}
group := map[string][]string{}
for _, d := range adds {
seen := map[string]bool{}
for _, s := range append(append([]string(nil), d.PublishAdded...), d.SubscribeAdded...) {
k := kindOfRight(s)
total++
byKind[k]++
if !seen[k] {
seen[k] = true
usersOf[k]++
}
}
t := tally(d)
group[t] = append(group[t], plainName(d.User))
}
summary = fmt.Sprintf("The grants part's list adds %s to %s and takes none away; no user, hash or login changes.", count(total, "right"), count(len(adds), "user"))
var kinds []string
for _, k := range rightKinds {
if byKind[k] > 0 {
kinds = append(kinds, fmt.Sprintf("%d %s (%s)", byKind[k], kindWords[k], count(usersOf[k], "user")))
}
}
// One point of the kinds, wrapped at the explanation's point length.
line := "Added: "
for i, k := range kinds {
if i > 0 && len([]rune(line))+len([]rune(k))+2 > 160 {
points = append(points, strings.TrimSuffix(line, ", "))
line = "Added: "
}
line += k + ", "
}
points = append(points, strings.TrimSuffix(line, ", ")+".")
if n := len(c.Excused); n > 0 {
points = append(points, fmt.Sprintf("Not counted: %d rights excused as before (a retired verb, a seat held on several machines).", n))
}
tallies := sortedKeys(group)
sort.SliceStable(tallies, func(i, j int) bool { return len(group[tallies[i]]) > len(group[tallies[j]]) })
var lines []string
used := 0
for i, t := range tallies {
users := group[t]
sort.Strings(users)
l := strings.Join(users, ", ") + ": " + t
if len(users) > 1 {
l += " each"
}
if used+len([]rune(l))+1 > AddedWordsBudget-80 {
rest := 0
for _, more := range tallies[i:] {
rest += len(group[more])
}
lines = append(lines, fmt.Sprintf("and %d more users: grants_user_list names every right added.", rest))
break
}
lines = append(lines, l)
used += len([]rune(l)) + 1
}
return summary, points, strings.Join(lines, "\n")
}
// count is n things, with the plural where n is not one.
func count(n int, thing string) string {
if n == 1 {
return "1 " + thing
}
return fmt.Sprintf("%d %ss", n, thing)
}
// Refusals are why a take-over is refused for the lists themselves, each naming its users: the loaded list unread, any user
// the lists keep apart other than by an added right (Removing), and any user with no hash in the vault but those both
// lists agree are gone. Empty when the lists differ at most by added rights and every login is kept.
func Refusals(c Comparison, cr Credentials) []string {
var reasons []string
if c.Unread != "" {
reasons = append(reasons, "the loaded list could not be read: "+c.Unread)
}
if d := c.Removing(); len(d) > 0 {
reasons = append(reasons, fmt.Sprintf("%d users differ between the shadow list and the loaded list other than by an added right: %s", len(d), strings.Join(d, ", ")))
}
var unhashed []string
for _, u := range append(append([]string(nil), cr.LeftOut...), cr.LosesLogin...) {
if !slices.Contains(c.Gone, u) {
unhashed = append(unhashed, u)
}
}
if len(unhashed) > 0 {
sort.Strings(unhashed)
reasons = append(reasons, fmt.Sprintf("the vault holds no hash for %d users (ADR 0547 clause 9: handed or adopted before take-over): %s",
len(unhashed), strings.Join(unhashed, ", ")))
}
return reasons
}
// TakeOverWords are the take-over ask's words for the lists as compared now: the rights the grants part's list adds
// (AddedWords) when nothing else keeps them apart, or else what a yes would be refused for, the users named in its list
// (ADR 0562 §2, §3). Empty when the lists agree. Held to the content rule: the loaded list's read error is not said.
func TakeOverWords(c Comparison, cr Credentials) (summary string, points []string, list string) {
refusals := Refusals(c, cr)
if len(refusals) == 0 {
return c.AddedWords()
}
summary = "Today a yes would be refused: the lists differ other than by an added right, the loaded list is unread, or a user has no hash."
var lines []string
if c.Unread != "" {
lines = append(lines, "The loaded list could not be read.")
refusals = refusals[1:]
}
used := 0
for _, r := range refusals {
if used+len([]rune(r)) > AddedWordsBudget-80 {
r = string([]rune(r)[:max(0, AddedWordsBudget-80-used)]) + "… grants_take_over_state says the rest."
}
lines = append(lines, r)
used += len([]rune(r)) + 1
if used >= AddedWordsBudget-80 {
break
}
}
for i := range lines {
if _, ok := outward.Check(lines[i]); !ok {
lines[i] = strings.ReplaceAll(lines[i], ".", " ")
}
}
return summary, []string{"Your yes now is refused and said, naming them; ask again once they agree."}, strings.Join(lines, "\n")
}
// plainName is a user's name as the operator channel's content rule lets it pass: as it is, or with spaces for its dots
// where it reads as a host name.
func plainName(user string) string {
if _, ok := outward.Check(user); ok {
return user
}
return strings.ReplaceAll(user, ".", " ")
}
// Compare compares the shadow list with the loaded one. placed says whether placement still names a loaded user's
// machine and module; nil names every one. Pure.
func Compare(shadow, loaded []User, placed func(user string) bool) Comparison {
modules/mesh-grants/leasecheck_test.go
-121
View File
@@ -1,121 +0,0 @@
package manifest
import (
"encoding/json"
"slices"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-grants/internal/grants"
)
// Issue 774, design 47 L3 (ADR 0547 §11, ADR 0561 §3, ADR 0571 §13): grants_lease_check answers from bus state who holds
// the controller's lease, every manifest's dependency on the controller with what keeps it, the controller's streams in
// use, and the lease write's check run dry, at the write (the three kept to it alone) and before its delivery. The
// judgement of each kind of dependency is mesh-sdk's and tested there (catalogue/controller_dependence_test.go, its rows
// moved from here, 774/S4); these rows are the tool's own: what it read, the lease, the streams.
func TestTheLeaseCheckJudgesEveryDependencyOnTheController(t *testing.T) {
const mManifest = `{"module":"mod-m","version":"1","reads":["mesh-controller.declared-packages"]}`
controllerHolds := map[string]any{"instance": "controller@novox-be pid 1 since 2026-10-30T14:52:10Z", "epoch": 60,
"taken": "2026-10-30T14:52:10Z", "renewed": "2026-10-30T22:00:00Z"}
handedAway := map[string]any{"handed-to": "the operator's hand-over", "handed-by": "operator"}
type want struct {
deps []string // "<module> <kind> <what> <kept?>" with kept as "kept" or "unkept"
writePasses bool
beforePasses bool
reason string // a reason the check at the write must name, or ""
bothPhases bool // the check before the delivery must name the reason too
handedAway bool
durable []string // durable consumers of KV_mesh-controller_assignments
}
for _, row := range []struct {
name string
manifests map[string]string
lease map[string]any
want want
setup func(b *fakeBus)
}{
{"a read of the lease alone is no dependency, and the controller's own manifest is not judged",
map[string]string{"mod-l": `{"module":"mod-l","version":"1","reads":["mesh-controller.lease"]}`,
"mesh-controller": `{"module":"mesh-controller","version":"1","invokes":["seat:mesh-controller.status"]}`},
controllerHolds, want{writePasses: true, beforePasses: true}, nil},
{"no manifest read from the indexer fails both phases, though nothing read depends on the controller",
map[string]string{},
controllerHolds, want{reason: "no manifest was read from the indexer (it holds 0 modules)", bothPhases: true}, nil},
{"a module whose manifest digest is missing fails both phases, named",
map[string]string{"mod-l": `{"module":"mod-l","version":"1"}`, "mod-m": mManifest},
controllerHolds, want{reason: "manifest of mod-m could not be read", bothPhases: true}, func(b *fakeBus) {
delete(b.states[grants.ReadManifests], delivery.ManifestDigest([]byte(mManifest)))
}},
{"a manifest the indexer's judge refuses fails both phases, named",
map[string]string{"mod-l": `{"module":"mod-l","version":"1"}`, "mod-r": `{"module":"mod-r","version":"1","waits":["mesh-controller.checked"]}`},
controllerHolds, want{reason: "manifest of mod-r could not be read", bothPhases: true}, func(b *fakeBus) {
b.refuse("mod-r", `{"module":"mod-r","version":"1","waits":["mesh-controller.checked"]}`, "waits on what no record keeps")
}},
{"a lease already handed away leaves nothing to write",
map[string]string{},
handedAway, want{handedAway: true, reason: "there is nothing to write: the lease is handed to the operator's hand-over"}, nil},
} {
t.Run(row.name, func(t *testing.T) {
b := newFakeBus()
for m, raw := range row.manifests {
b.manifestOf(m, raw)
}
b.put(grants.ReadLease, grants.LeaseKey, row.lease)
if row.setup != nil {
row.setup(b)
}
b.streams = []map[string]any{
{"name": "KV_mesh-controller_assignments", "consumers": 3, "messages": 2991, "subjects_held": 40, "last": "2026-10-30T12:55:13Z"},
{"name": "KV_mesh-controller_rollouts", "consumers": 0, "messages": 6652},
{"name": "KV_mesh-grants_memberships", "consumers": 243, "messages": 1573},
}
b.consumers = map[string][]map[string]any{"KV_mesh-controller_assignments": {{"name": "0AYfjxta"}, {"name": "1g9XJFUX"}, {"name": "grants-assignments"}}}
raw, err := partOn(b).LeaseCheck()
if err != nil {
t.Fatal(err)
}
var got grants.LeaseAnswer
data, _ := json.Marshal(raw)
if err := json.Unmarshal(data, &got); err != nil {
t.Fatal(err)
}
var deps []string
for _, d := range got.Dependencies {
kept := "unkept"
if d.Kept != "" {
kept = "kept"
}
deps = append(deps, d.Module+" "+d.Kind+" "+d.What+" "+kept)
}
if !slices.Equal(deps, row.want.deps) {
t.Errorf("dependencies %q, want %q", deps, row.want.deps)
}
if got.Check.Phase != grants.PhaseWrite || got.Check.Passes != row.want.writePasses {
t.Errorf("check at the write passes=%v (%v), want %v", got.Check.Passes, got.Check.Reasons, row.want.writePasses)
}
if got.Before.Phase != grants.PhaseBefore || got.Before.Passes != row.want.beforePasses {
t.Errorf("check before the delivery passes=%v (%v), want %v", got.Before.Passes, got.Before.Reasons, row.want.beforePasses)
}
if row.want.reason != "" && !slices.ContainsFunc(got.Check.Reasons, func(r string) bool { return strings.Contains(r, row.want.reason) }) {
t.Errorf("reasons %q name nothing like %q", got.Check.Reasons, row.want.reason)
}
if row.want.bothPhases && !slices.ContainsFunc(got.Before.Reasons, func(r string) bool { return strings.Contains(r, row.want.reason) }) {
t.Errorf("reasons before the delivery %q name nothing like %q", got.Before.Reasons, row.want.reason)
}
if got.Lease.HandedAway != row.want.handedAway {
t.Errorf("lease handed away %v (%s), want %v", got.Lease.HandedAway, got.Lease.Says, row.want.handedAway)
}
if !row.want.handedAway && (got.Lease.Since == nil || got.Lease.Since.Format("15:04") != "14:52" ||
!strings.Contains(got.Lease.Says, "the controller holds the lease") || !slices.Equal(got.Lease.Does, grants.DoesWhileController)) {
t.Errorf("lease %+v: the controller's instance since 14:52, and what it does while it holds it", got.Lease)
}
if len(got.Streams) != 1 || got.Streams[0].Stream != "KV_mesh-controller_assignments" || got.Streams[0].Consumers != 3 ||
!slices.Equal(got.Streams[0].Durable, []string{"grants-assignments"}) || !slices.Equal(got.Idle, []string{"KV_mesh-controller_rollouts"}) {
t.Errorf("streams %+v idle %v: the controller's assignments bucket with its one durable consumer, the rollouts idle, no other module's", got.Streams, got.Idle)
}
})
}
}
modules/mesh-grants/leasewrite_test.go
-503
View File
@@ -1,503 +0,0 @@
package manifest
import (
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-sdk/go/seal"
"mesh-grants/internal/grants"
)
// leaseFake is the bus as the lease write meets it: fakeBus's states and tools, and the seats and tools the write asks.
// Each answer is shaped as its writer's code answers it (issue 774): node-service-manager's status the systemd module's
// Manager.Status (ActiveState), nats_bucket_expiry nats-tools' BucketExpirySet (now), nats_consumer_remove its
// ConsumerRemoved (removed) and an absent consumer its error ("has no consumer"), nats_stream_remove its StreamRemoved
// (removed, present), issue-tracker.get mesh-issues' Brief (status), node_trusts mesh-delivery-view's TrustsAnswer
// (every), mesh-config's compare its Comparison (source, count), mesh-wants' upgrade compare the assigner's answer
// (empty), vault_status mesh-vault's answer (ready, minting.on, minting.held_back).
type leaseFake struct {
*fakeBus
answer map[string]func(map[string]any) (any, error)
active string // the controller's ActiveState
expiry time.Duration // the lease bucket's
ttls map[string]time.Duration
log []string // the write's acts on the bus, in order
logged []string // the process's log lines
}
func newLeaseFake(t *testing.T) (*leaseFake, *grants.Part) {
t.Helper()
b := fixture()
b.manifestOf("nats", `{"module":"nats","version":"1","claims":[{"name":"mesh-broker"}],"own-secrets":{"broker":"/var/lib/nats/broker"},`+
`"tools":["nats_bucket_expiry","nats_consumer_remove","nats_stream_remove"]}`)
// The lease's host is the machine's hostname, never a node's name (issue 774: "novox-be" on the node novox); the
// controller's node is the one placement places it on.
b.put(grants.ReadLease, grants.LeaseKey, map[string]any{"instance": "controller@box-a pid 1", "host": "box-a", "epoch": 60,
"taken": "2026-10-30T14:52:10Z"})
b.assign("a", grants.Controller)
b.put(grants.ReadTakeover, health.TakeoverKey, health.Takeover{V: 1, Follow: health.FollowKeeper, By: "operator", Why: "the keeper took over"})
busKey, err := seal.GenerateKeyPair()
if err != nil {
t.Fatal(err)
}
b.put(grants.ReadHave, "a.nats", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK,
UserList: &delivery.UserListWritten{Source: "a.nats.src2", Revision: 7, Digest: "sha256:aa", Users: 6, Key: busKey.PublicKey}})
// The controller's hash in the vault, so the list carries the controller and the cut changes it.
b.put(grants.ReadAccounts, buscompose.ControllerName, map[string]any{"hash": "$2a$11$controller", "generation": 1,
"at": "2026-10-30T14:52:10Z"})
f := &leaseFake{fakeBus: b, active: "active", expiry: 15 * time.Second, ttls: map[string]time.Duration{}}
f.answer = map[string]func(map[string]any) (any, error){
grants.CallIssue: func(map[string]any) (any, error) { return map[string]any{"status": "resolved"}, nil },
grants.CallNodeTrusts: func(map[string]any) (any, error) { return map[string]any{"every": true, "not_yet": []string{}}, nil },
grants.CallConfigCompare: func(map[string]any) (any, error) { return map[string]any{"source": "indexer", "count": 0}, nil },
grants.CallUpgrade: func(map[string]any) (any, error) { return map[string]any{"empty": true, "splits": []any{}}, nil },
grants.CallVaultStatus: func(map[string]any) (any, error) {
return map[string]any{"ready": true, "minting": map[string]any{"on": true, "held_back": map[string]string{}}}, nil
},
grants.CallServiceStop: func(map[string]any) (any, error) { f.active = "inactive"; return map[string]any{"ok": true}, nil },
grants.CallServiceStart: func(map[string]any) (any, error) { f.active = "active"; return map[string]any{"ok": true}, nil },
grants.CallServiceStatus: func(map[string]any) (any, error) { return map[string]any{"ActiveState": f.active}, nil },
grants.CallBucketExpiry: func(a map[string]any) (any, error) {
now := fmt.Sprint(a["max_age"])
d, _ := time.ParseDuration(strings.Replace(now, "none", "0s", 1))
f.expiry = d
f.log = append(f.log, "expiry "+now)
return map[string]any{"bucket": a["bucket"], "now": now, "changed": true}, nil
},
grants.CallRecordWarrant: func(a map[string]any) (any, error) { f.log = append(f.log, "warranted"); return map[string]any{}, nil },
grants.CallRecordHandAct: func(a map[string]any) (any, error) {
f.log = append(f.log, "hand-act "+fmt.Sprint(a["what"]))
return map[string]any{}, nil
},
grants.CallConsumerRemove: func(map[string]any) (any, error) {
f.log = append(f.log, "consumer removed")
return map[string]any{"removed": true}, nil
},
grants.CallStreamRemove: func(map[string]any) (any, error) {
f.log = append(f.log, "stream removed")
return map[string]any{"removed": true, "present": true}, nil
},
}
p := partOn(b)
p.Call, p.Store, p.Lease, p.Asker, p.Objects = f, ttlStore{ownStore{b}, f}, f, &fakeAsker{wheres: map[string]*asks.Warrant{}}, &fakeObjects{}
p.SaySeat = func(event string, _ any) error { f.log = append(f.log, "event "+event); return nil }
p.Nonce = func() string { return "0001" }
p.Logf = func(format string, a ...any) { f.logged = append(f.logged, fmt.Sprintf(format, a...)) }
return f, p
}
func (f *leaseFake) Call(key string, body any) (json.RawMessage, error) {
base, _, _ := strings.Cut(key, "@")
if answer, ok := f.answer[base]; ok {
f.mu.Lock()
f.calls = append(f.calls, key)
f.mu.Unlock()
args, _ := body.(map[string]any)
out, err := answer(args)
if err != nil {
return nil, err
}
return json.Marshal(out)
}
return f.fakeBus.Call(key, body)
}
func (f *leaseFake) PutHolder(value []byte) error {
f.log = append(f.log, "holder")
f.states[grants.ReadLease][grants.LeaseKey] = value
return nil
}
func (f *leaseFake) DeleteHolder() error {
f.log = append(f.log, "holder deleted")
delete(f.states[grants.ReadLease], grants.LeaseKey)
return nil
}
func (f *leaseFake) Expiry() (time.Duration, error) { return f.expiry, nil }
// loads says the bus node loaded the list the part last sealed.
func (f *leaseFake) loads(p *grants.Part) {
g, _, _ := p.Sealed()
f.put(grants.ReadHave, "a.nats", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK,
UserList: &delivery.UserListWritten{Source: delivery.GrantsUserListSource, Revision: g.Revision, Digest: g.Digest,
Users: g.Users, Key: keyOf(f)}})
}
func keyOf(f *leaseFake) string {
var h delivery.Have
_ = json.Unmarshal(f.states[grants.ReadHave]["a.nats"], &h)
return h.UserList.Key
}
// ttlStore is the part's own store with a time to live per key, recorded.
type ttlStore struct {
ownStore
f *leaseFake
}
func (s ttlStore) PutTTL(state, key string, value []byte, ttl time.Duration) (uint64, error) {
s.f.ttls[state+"."+key] = ttl
s.f.log = append(s.f.log, key+" "+ttl.String())
return s.Put(state, key, value)
}
// record is the part's lease record now.
func record(t *testing.T, p *grants.Part) grants.LeaseRecord {
t.Helper()
r, err := p.LeaseState()
if err != nil {
t.Fatal(err)
}
return r
}
// warrant answers the open act's ask with option, as the router's warrant hands it.
func warrant(t *testing.T, p *grants.Part, option string) error {
t.Helper()
r := record(t, p)
if r.Open == nil {
t.Fatal("no act is open")
}
return p.Warranted(asks.Option{ID: option}, asks.Warrant{Ask: r.Open.Ask, Option: option, Outcome: asks.OutcomeChosen})
}
// Issue 774 (novox/hq ADR 0582): the lease write, from its ask through the day-later removal, against stand-in seats.
func TestTheLeaseWrite(t *testing.T) {
// 774/S14/AC1: any one precondition not green refuses the verb naming it; all green asks.
t.Run("preconditions", func(t *testing.T) {
for _, row := range []struct {
name string
break_ func(f *leaseFake)
}{
{"lease-check", func(f *leaseFake) {
f.manifestOf("mod-u", `{"module":"mod-u","version":"1","reads":["mesh-controller.conditions"]}`)
}},
{"earlier-steps", func(f *leaseFake) {
f.answer[grants.CallIssue] = func(a map[string]any) (any, error) {
if fmt.Sprint(a["number"]) == "1954" {
return map[string]any{"status": "fixing"}, nil
}
return map[string]any{"status": "resolved"}, nil
}
}},
{"node-trusts", func(f *leaseFake) {
f.answer[grants.CallNodeTrusts] = func(map[string]any) (any, error) {
return map[string]any{"every": false, "not_yet": []string{"ace"}}, nil
}
}},
{"config-compare", func(f *leaseFake) {
f.answer[grants.CallConfigCompare] = func(map[string]any) (any, error) { return map[string]any{"source": "indexer", "count": 2}, nil }
}},
{"upgrade-compare", func(f *leaseFake) {
f.answer[grants.CallUpgrade] = func(map[string]any) (any, error) { return map[string]any{"empty": false, "splits": []any{1}}, nil }
}},
{"keeper-take-over", func(f *leaseFake) {
f.put(grants.ReadTakeover, health.TakeoverKey, health.Takeover{V: 1, Follow: health.FollowController})
}},
{"vault-minting", func(f *leaseFake) {
f.answer[grants.CallVaultStatus] = func(map[string]any) (any, error) {
return map[string]any{"ready": true, "minting": map[string]any{"on": true, "held_back": map[string]string{"a.x": "why"}}}, nil
}
}},
{"bus-tools", func(f *leaseFake) { f.manifestOf("nats", natsManifest) }},
{"", func(*leaseFake) {}},
} {
f, p := newLeaseFake(t)
row.break_(f)
_, err := p.LeaseWrite("operator@shanks", "the controller goes")
switch {
case row.name == "" && err != nil:
t.Errorf("all green: refused: %v", err)
case row.name != "" && (err == nil || !strings.Contains(err.Error(), row.name+": ")):
t.Errorf("%s not green: %v, want a refusal naming it", row.name, err)
case row.name != "":
for _, other := range grants.LeaseCheckNames {
if other != row.name && strings.Contains(err.Error(), other+": ") {
t.Errorf("%s not green: the refusal names %s too: %v", row.name, other, err)
}
}
}
if r := record(t, p); (r.Open != nil) != (row.name == "") {
t.Errorf("%q: an act open %v", row.name, r.Open)
}
}
})
// 774/S15/AC1, AC2: a no, an expiry, or a warrant whose checks changed writes nothing, naming the changed check.
t.Run("the ask and the warrant", func(t *testing.T) {
for _, row := range []struct {
name, step, said string
answer func(t *testing.T, f *leaseFake, p *grants.Part)
}{
{"declined", grants.StepDeclined, "said no", func(t *testing.T, _ *leaseFake, p *grants.Part) {
if err := warrant(t, p, "no"); err != nil {
t.Fatal(err)
}
}},
{"expired", "expired", "expired", func(t *testing.T, _ *leaseFake, p *grants.Part) {
r := record(t, p)
p.Said("", asks.Warrant{Ask: r.Open.Ask, Outcome: asks.OutcomeExpired}, nil)
}},
{"checks changed", grants.StepRefused, "the checks changed since the ask: vault-minting (now: ready true, minting true, 1 held back)",
func(t *testing.T, f *leaseFake, p *grants.Part) {
// The vault holds one account back between the ask and the warrant.
f.answer[grants.CallVaultStatus] = func(map[string]any) (any, error) {
return map[string]any{"ready": true, "minting": map[string]any{"on": true, "held_back": map[string]string{"a.x": "why"}}}, nil
}
if err := warrant(t, p, "yes"); err != nil {
t.Fatal(err)
}
}},
} {
f, p := newLeaseFake(t)
if _, err := p.LeaseWrite("operator@shanks", "the controller goes"); err != nil {
t.Fatal(err)
}
p.Compose() // the process asks it
if a := p.Asker.(*fakeAsker); len(a.asks) != 1 || len(a.asks[0].Options) != 2 || a.asks[0].Options[0].Binds == "" {
t.Fatalf("%s: asked %+v, want one ask whose yes binds the act", row.name, a.asks)
}
row.answer(t, f, p)
r := record(t, p)
if r.Open != nil || len(r.Acts) != 1 || r.Acts[0].Step != row.step || !strings.Contains(r.Acts[0].Said, row.said) {
t.Errorf("%s: open %v, acts %+v; want one act %s saying %q", row.name, r.Open, r.Acts, row.step, row.said)
}
if f.count(grants.CallServiceStop+"@a") != 0 || len(f.log) != 0 || r.Cut {
t.Errorf("%s: something was written: stops %d, %v, cut %v", row.name, f.count(grants.CallServiceStop+"@a"), f.log, r.Cut)
}
}
})
// 774/S16/AC1, AC2; 774/S17/AC1, AC2: the write's six steps, in order, each read back before the next.
t.Run("the write", func(t *testing.T) {
f, p := newLeaseFake(t)
f.answer[grants.CallServiceStop] = func(map[string]any) (any, error) { return map[string]any{"ok": true}, nil } // reads back active
if _, err := p.LeaseWrite("operator@shanks", "the controller goes"); err != nil {
t.Fatal(err)
}
p.Compose()
if err := warrant(t, p, "yes"); err != nil {
t.Fatal(err)
}
if r := record(t, p); r.Open != nil || r.Cut || r.Acts[0].Step != grants.StepFailed || !strings.Contains(r.Acts[0].Said, "did not read back stopped") || len(f.log) != 0 {
t.Fatalf("a stop not read back: %+v, %v; want the act failed there, nothing written", r, f.log)
}
f, p = newLeaseFake(t)
if _, err := p.LeaseWrite("operator@shanks", "the controller goes"); err != nil {
t.Fatal(err)
}
p.Compose()
before, _, _ := p.Sealed()
if err := warrant(t, p, "yes"); err != nil {
t.Fatal(err)
}
r := record(t, p)
g, _, _ := p.Sealed()
if f.active != "inactive" || !r.Cut || r.Open == nil || r.Open.Step != grants.StepCut || r.Open.List != g.Digest || g.Digest == before.Digest {
t.Fatalf("after the warrant: active %s, %+v, sealed %s (was %s); want stopped, cut and waiting on the cut list", f.active, r, g.Digest, before.Digest)
}
if len(f.log) != 0 {
t.Fatalf("the expiry or the key was touched before the cut list loaded: %v", f.log)
}
// The cut list takes the controller's write on the lease and gives mesh-grants the key holder.
var st grants.UserListState
raw, _, _, _ := ownStore{f.fakeBus}.Get(grants.StateUserList, grants.KeyList)
_ = json.Unmarshal(raw, &st)
p.Compose() // a pass with the bus node not loaded yet waits
if len(f.log) != 0 {
t.Fatalf("a pass before the load acted: %v", f.log)
}
f.loads(p)
p.Compose()
want := []string{"expiry none", "holder", "event lease-handed", "warranted", "removal-due 24h0m0s"}
if !slices.Equal(f.log, want) {
t.Errorf("the write's acts %v, want %v in that order", f.log, want)
}
var h buscompose.LeaseHolder
_ = json.Unmarshal(f.states[grants.ReadLease][grants.LeaseKey], &h)
r = record(t, p)
if h.HandedTo != "mesh-grants" || h.HandedBy != "operator" || h.Why != "the controller goes" || h.Ask != r.Ask || h.At.IsZero() ||
r.Open != nil || r.Handed.IsZero() || r.Acts[len(r.Acts)-1].Step != grants.StepDone {
t.Errorf("holder %+v, record %+v", h, r)
}
p.Compose() // the pass the lease key's change wakes
if g, _, _ := p.Sealed(); !g.LeaseHanded {
t.Error("the user-list key does not say the lease is handed")
}
if _, err := p.HandBack("operator@shanks", "back"); err == nil || !strings.HasPrefix(err.Error(), "gone") {
t.Errorf("the user list's hand-back once the lease is handed: %v, want gone", err)
}
if _, err := p.LeaseWrite("operator@shanks", "again"); err == nil {
t.Error("a second lease-write with the lease handed was asked")
}
// 774/S18/AC1: the key or the expiry changed by another hand is said once, and nothing is written.
f.log = nil
f.put(grants.ReadLease, grants.LeaseKey, map[string]any{"instance": "controller@a pid 9", "host": "box-a", "epoch": 61})
p.Compose()
p.Compose()
f.expiry = 15 * time.Second
p.Compose()
if !slices.Equal(f.log, []string{"event lease-altered"}) || !strings.Contains(record(t, p).Altered, "controller@a pid 9") {
t.Errorf("an altered lease said %v (%q), want lease-altered once naming it", f.log, record(t, p).Altered)
}
})
// Issue 774 (the lease write met at novox): the controller's service is stopped on the node placement places
// mesh-controller on, never on the lease's host (a hostname); none or two such nodes refuse by name, and a failed act
// is logged beside its condition.
t.Run("the controller's node", func(t *testing.T) {
for _, row := range []struct {
name, stop, said string
place func(f *leaseFake)
}{
{"placed on a", "a", "", func(*leaseFake) {}},
{"placed nowhere", "", "no node places mesh-controller", func(f *leaseFake) {
delete(f.states[grants.ReadPlacement], "placement.a")
f.assign("a", "nats")
f.assign("a", "mod-x")
}},
{"placed twice", "", "two or more nodes place mesh-controller (a, b)", func(f *leaseFake) { f.assign("b", grants.Controller) }},
} {
f, p := newLeaseFake(t)
if _, err := p.LeaseWrite("operator@shanks", "the controller goes"); err != nil {
t.Fatal(err)
}
p.Compose()
row.place(f)
if err := warrant(t, p, "yes"); err != nil {
t.Fatal(err)
}
stops := slices.DeleteFunc(slices.Clone(f.calls), func(c string) bool { return !strings.HasPrefix(c, grants.CallServiceStop+"@") })
r := record(t, p)
switch {
case row.stop != "" && (!slices.Equal(stops, []string{grants.CallServiceStop + "@" + row.stop}) || r.Node != row.stop):
t.Errorf("%s: stopped %v, node %q; want the stop on %s alone", row.name, stops, r.Node, row.stop)
case row.stop == "" && (len(stops) != 0 || r.Open != nil || len(r.Acts) != 1 || r.Acts[0].Step != grants.StepFailed ||
!strings.Contains(r.Acts[0].Said, row.said)):
t.Errorf("%s: stopped %v, %+v; want the act failed saying %q, nothing stopped", row.name, stops, r, row.said)
case row.stop == "" && !slices.ContainsFunc(f.logged, func(l string) bool { return strings.Contains(l, row.said) }):
t.Errorf("%s: logged %q, want a line saying %q", row.name, f.logged, row.said)
}
}
})
// 774/S19/AC1, AC2: hand-back restores the expiry, deletes holder and removal-due, and starts the controller only once
// the list with its rights is loaded; once the removal ran it is refused.
t.Run("hand-back", func(t *testing.T) {
f, p := handed(t)
if _, err := p.LeaseHandBack("operator@shanks", "it goes back"); err != nil {
t.Fatal(err)
}
p.Compose()
f.log = nil
if err := warrant(t, p, "yes"); err != nil {
t.Fatal(err)
}
r := record(t, p)
if f.expiry != 15*time.Second || f.states[grants.ReadLease][grants.LeaseKey] != nil || r.Cut || f.active != "inactive" {
t.Fatalf("after the warrant: expiry %s, holder %s, cut %v, controller %s", f.expiry, f.states[grants.ReadLease][grants.LeaseKey], r.Cut, f.active)
}
if _, _, found, _ := f.ownGet(grants.StateLease, grants.KeyRemovalDue); found {
t.Error("removal-due is not deleted")
}
f.loads(p)
p.Compose()
if r := record(t, p); f.active != "active" || r.Open != nil || r.Acts[len(r.Acts)-1].Step != grants.StepDone {
t.Errorf("once loaded: controller %s, %+v", f.active, r)
}
// AC2: once the removal ran, refused naming why.
f, p = handed(t)
p.RemovalDueHeard()
p.Compose()
if _, err := p.LeaseHandBack("operator@shanks", "too late"); err == nil || !strings.Contains(err.Error(), "without its consumer") {
t.Errorf("hand-back after the removal: %v", err)
}
})
// 774/S20/AC1, AC2, AC3: the marker of removal-due starts the removal; a failure says removal-failed and writes the key
// again with a day; a hand-back's delete removes nothing.
t.Run("the day-later removal", func(t *testing.T) {
f, p := handed(t)
f.log = nil
p.RemovalDueHeard()
p.Compose()
want := []string{"consumer removed", "hand-act removed the controller's consumer of DEAD_LETTER_NOTICES", "stream removed",
"hand-act removed the stream ASSIGNMENTS"}
if !slices.Equal(f.log, want) || record(t, p).Removed.IsZero() {
t.Errorf("the removal %v (removed %v), want %v", f.log, record(t, p).Removed, want)
}
f, p = handed(t)
f.log = nil
f.answer[grants.CallStreamRemove] = func(map[string]any) (any, error) {
return nil, errors.New("ASSIGNMENTS has 1 consumers: a reader is attached, so it was not removed")
}
p.RemovalDueHeard()
p.Compose()
if !slices.Contains(f.log, "event removal-failed") || f.ttls[grants.StateLease+"."+grants.KeyRemovalDue] != grants.RemovalAfter ||
!record(t, p).Removed.IsZero() {
t.Errorf("a failed removal: %v, removal-due %s", f.log, f.ttls[grants.StateLease+"."+grants.KeyRemovalDue])
}
f, p = handed(t)
if _, err := p.LeaseHandBack("operator@shanks", "it goes back"); err != nil {
t.Fatal(err)
}
p.Compose()
if err := warrant(t, p, "yes"); err != nil {
t.Fatal(err)
}
f.log = nil
p.RemovalDueHeard() // the hand-back's own delete of removal-due, heard
p.Compose()
if slices.ContainsFunc(f.log, func(s string) bool { return strings.Contains(s, "removed") }) {
t.Errorf("a hand-back's delete of removal-due removed %v", f.log)
}
})
// 774/S21/AC1: once the lease is handed away a membership is written to the state and not published on its subject.
t.Run("memberships after the write", func(t *testing.T) {
f, p := handed(t)
f.publishes = nil
f.assign("b", "mod-x")
p.Compose()
if len(f.publishes) != 0 {
t.Errorf("published on mesh.assignment after the lease write: %v", f.publishes)
}
if _, _, found, _ := f.ownGet(grants.StateMemberships, "b.mod-x"); !found {
t.Error("the membership is not written to the state")
}
})
}
// handed is a part whose lease write is done: the lease names mesh-grants and removal-due is written.
func handed(t *testing.T) (*leaseFake, *grants.Part) {
t.Helper()
f, p := newLeaseFake(t)
if _, err := p.LeaseWrite("operator@shanks", "the controller goes"); err != nil {
t.Fatal(err)
}
p.Compose()
if err := warrant(t, p, "yes"); err != nil {
t.Fatal(err)
}
f.loads(p)
p.Compose()
if r := record(t, p); r.Handed.IsZero() {
t.Fatalf("the write did not finish: %+v, %v", r, f.log)
}
return f, p
}
modules/mesh-grants/module.json
+3 -40
View File
@@ -2,16 +2,6 @@
"module": "mesh-grants",
"version": "1",
"slug": "grants",
"claims": [
{
"name": "mesh-grants",
"scope": "mesh",
"serves": [
"lease-write",
"lease-hand-back"
]
}
],
"reads": [
"mesh-config.resolved",
"mesh-delivery-assigner.wants",
@@ -29,8 +19,7 @@
"consumes": [
"mesh-conditions.condition-raised",
"mesh-conditions.condition-cleared",
"mesh-vault.account-issued",
"nats.bucket-expiry-set"
"mesh-vault.account-issued"
],
"uses": [
"operator-channel"
@@ -39,20 +28,7 @@
"nats.nats_streams",
"nats.nats_user_can",
"nats.nats_users",
"seat:node-engine.root-free",
"seat:mesh-hand-acts.warranted",
"seat:node-service-manager.stop",
"seat:node-service-manager.start",
"seat:node-service-manager.status",
"nats.nats_bucket_expiry",
"nats.nats_consumer_remove",
"nats.nats_stream_remove",
"seat:mesh-hand-acts.hand-act",
"mesh-delivery-view.node_trusts",
"seat:mesh-config.compare",
"seat:mesh-wants.upgrade",
"mesh-vault.vault_status",
"seat:issue-tracker.get"
"seat:node-engine.root-free"
],
"emits": [
"asserted",
@@ -67,14 +43,9 @@
"grants_user_list",
"grants_compose_now",
"grants_assert",
"grants_take_over",
"grants_take_over_state",
"grants_hand_back",
"grants_accounts",
"grants_consumer_match",
"grants_accept_hashes",
"grants_lease_check",
"grants_removal_due"
"grants_accept_hashes"
],
"state": [
{
@@ -89,18 +60,10 @@
"name": "grants",
"history": 5
},
{
"name": "lease",
"key-ttl": true
},
{
"name": "memberships",
"history": 5
},
{
"name": "takeover",
"history": 20
},
{
"name": "user-list",
"history": 20