novox/mesh-delivery
19
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request 'The grants part and the indexer without the controller; consumers far behind raise a condition (issue 1126 group B)' (#138) from feat/1126-removal into main

Browse Source
This commit was merged in pull request #138.
This commit is contained in:
mesh-tools
2026-11-02 13:41:16 +00:00
parent 4dd7dcfc14 e2cb398186
commit 511e4b7414
93 changed files with 989 additions and 4830 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compose_integration_test.go
+8 -8
View File
@@ -91,7 +91,7 @@ func (b *bed) hold(t *testing.T) *holder {
// 1622/AC1-AC3 and 1623 on the bus: started whole, the assigner composes a group holding a mesh-sdk pull request by
// asking the Go build seat for the composer at the member's head, and for the composer at the mesh-sdk the running
// controller pins; it waits naming the asks while they run, writes the verdict from their answers naming the mesh-sdk the
// node-engine pins; it waits naming the asks while they run, writes the verdict from their answers naming the mesh-sdk the
// head's composer was built with, and asks nothing more on a later pass of the same heads.
func TestAGroupWithAnSDKMemberIsComposedByTheComposerAtItsHead(t *testing.T) {
b := startBed(t, "")
@@ -104,8 +104,8 @@ func TestAGroupWithAnSDKMemberIsComposedByTheComposerAtItsHead(t *testing.T) {
b.putJSON(t, delivery.ModulesBucket, "web", mod)
b.place(t, "g14", "web")
running := strings.Repeat("3", 40)
b.place(t, "novox", "mesh-controller")
b.built(t, "mesh-controller", "novox/mesh-controller", running)
b.place(t, "novox", "mesh-host")
b.built(t, "mesh-host", "novox/mesh-host", running)
sdkHead := strings.Repeat("4", 40)
heads := []delivery.Head{{Repository: "novox/mesh-sdk", Number: 80, Head: sdkHead}, {Repository: "novox/web", Number: 7, Head: strings.Repeat("2", 40)}}
hash, _ := delivery.HeadsHash(heads)
@@ -121,8 +121,8 @@ func TestAGroupWithAnSDKMemberIsComposedByTheComposerAtItsHead(t *testing.T) {
key, _ := delivery.CompositionKey("feat/x", hash)
// No holder takes the asks yet: the composition waits, naming them.
waits := b.verdict(t, key, func(v delivery.CompositionVerdict) bool { return v.State == delivery.CompositionWaits })
if !strings.Contains(waits.Words, "mesh-sdk "+sdkHead) || !strings.Contains(waits.Words, "mesh-controller at "+running) {
t.Fatalf("the waiting composition says %q, want the head's and the running controller's asks", waits.Words)
if !strings.Contains(waits.Words, "mesh-sdk "+sdkHead) || !strings.Contains(waits.Words, "mesh-host at "+running) {
t.Fatalf("the waiting composition says %q, want the head's and the running node-engine's asks", waits.Words)
}
h := b.hold(t)
v := b.verdict(t, key, func(v delivery.CompositionVerdict) bool { return v.State != delivery.CompositionWaits })
@@ -131,7 +131,7 @@ func TestAGroupWithAnSDKMemberIsComposedByTheComposerAtItsHead(t *testing.T) {
}
asks := h.taken()
if len(asks) != 2 {
t.Fatalf("the seat was asked %d composes, want two (the head, the running controller)", len(asks))
t.Fatalf("the seat was asked %d composes, want two (the head, the running node-engine)", len(asks))
}
byRepo := map[string]delivery.StepAsk{}
for _, a := range asks {
@@ -143,8 +143,8 @@ func TestAGroupWithAnSDKMemberIsComposedByTheComposerAtItsHead(t *testing.T) {
if a := byRepo["novox/mesh-sdk"]; a.Compose == nil || a.Compose.Fetch["commit"] != sdkHead || a.Compose.Dir != "go" {
t.Errorf("the head's ask = %+v", a.Compose)
}
if a := byRepo["novox/mesh-controller"]; a.Compose == nil || a.Compose.Fetch["commit"] != running || a.Compose.Dir != delivery.RootPath {
t.Errorf("the running controller's ask = %+v", a.Compose)
if a := byRepo["novox/mesh-host"]; a.Compose == nil || a.Compose.Fetch["commit"] != running || a.Compose.Dir != delivery.RootPath {
t.Errorf("the running node-engine's ask = %+v", a.Compose)
}
// A later pass of the same heads (woken by any change) asks nothing more.
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go
+28 -29
View File
@@ -26,6 +26,7 @@ import (
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/compose/judge"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-assigner/internal/assign"
@@ -419,8 +420,8 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
bkey, _ := delivery.BuilderBuildKey("web", fNew)
b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
for _, fp := range []string{fNew, fOld} {
gkey, _ := delivery.ControllerGrantKey("web", fp)
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
gkey, _ := delivery.PartGrantKey("web", fp)
grant, _ := delivery.EncodeGrant(delivery.Grant{})
b.putJSON(t, delivery.PartGrantsBucket, gkey, grant)
}
runs := map[string]string{"ace": fOld, "g14": "build-9", "shanks": fOld}
@@ -446,12 +447,12 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
var neither asks.Condition
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && neither.Key == ""; time.Sleep(100 * time.Millisecond) {
for k, c := range b.conditionsSaid(t) {
if strings.Contains(k, ".neither.") && c.State == asks.StateRaised {
if strings.HasSuffix(k, ".neither") && c.State == asks.StateRaised {
neither = c
}
}
}
if neither.Subject.Machine != "g14" || neither.Subject.ID != "web" || neither.Severity != asks.SeverityUrgent || !strings.Contains(neither.Explanation, "canary") {
if neither.Subject.Machine != "g14" || !strings.HasPrefix(neither.Subject.ID, "g14.web") || neither.Severity != asks.SeverityUrgent || !strings.Contains(neither.Explanation, "canary") {
t.Errorf("the condition of web on neither list on g14 = %+v", neither)
}
if keys, err := b.buckets[delivery.AssignerDifferencesBucket].Keys(b.ctx); err == nil && len(keys) != 0 {
@@ -532,36 +533,34 @@ func TestAHeldFingerprintIsAskedAbout(t *testing.T) {
}
}
// conditionsSaid reads what the assigner said on the operator channel's subject for conditions, by condition key.
// conditionsSaid reads what the assigner said of its conditions, by finding key: each a finding in its own bucket, which
// the condition keeper carries (novox/hq ADR 0590 §3, issue 1126), raised while it stands and cleared once deleted, each
// with the words its last finding said and the subject the finding names (`<module>[.<fingerprint>]`).
func (b *bed) conditionsSaid(t *testing.T) map[string]asks.Condition {
t.Helper()
stream, err := b.js.Stream(b.ctx, "CONDITIONS")
w, err := b.buckets[assign.FindingsBucket].WatchAll(b.ctx, jetstream.IncludeHistory())
if err != nil {
t.Fatal(err)
}
defer w.Stop()
out := map[string]asks.Condition{}
cons, err := stream.CreateOrUpdateConsumer(b.ctx, jetstream.ConsumerConfig{AckPolicy: jetstream.AckNonePolicy})
if err != nil {
t.Fatal(err)
}
for {
batch, err := cons.FetchNoWait(100)
if err != nil {
t.Fatal(err)
for e := range w.Updates() {
if e == nil {
return out
}
n := 0
for m := range batch.Messages() {
n++
c, err := asks.DecodeCondition(m.Data())
c := out[e.Key()]
c.Key, c.State = e.Key(), asks.StateCleared
if e.Operation() == jetstream.KeyValuePut {
f, err := health.DecodeFinding(e.Value())
if err != nil {
t.Fatal(err)
}
out[c.Key] = c
}
if n == 0 {
return out
c = asks.Condition{Key: e.Key(), State: asks.StateRaised, Severity: string(f.Severity), Explanation: f.Explanation,
Resolved: f.Resolved, Subject: asks.Scope{Scope: string(f.Subject.Scope), ID: f.Subject.ID, Machine: f.Subject.Machine}}
}
out[e.Key()] = c
}
return out
}
// 1906/D3 (ADR 0539 §19), whole on the bus: a module holding data whose new fingerprint failed its health check on the
@@ -594,7 +593,7 @@ func TestAFailedDataModuleIsAskedToRestore(t *testing.T) {
key := ""
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && key == ""; time.Sleep(100 * time.Millisecond) {
for k, c := range b.conditionsSaid(t) {
if c.State == asks.StateRaised && c.Subject.ID == "mail" && strings.Contains(k, ".held.") {
if c.State == asks.StateRaised && strings.HasPrefix(c.Subject.ID, "mail.") && strings.HasSuffix(k, ".held") {
if c.Subject.Machine != "ace" || !strings.Contains(c.Explanation, "connection refused") {
t.Fatalf("the condition = %+v, want it to name ace and the check's words", c)
}
@@ -629,8 +628,8 @@ func TestTheAssignerReplacedResumesWithNothingLostOrRepeated(t *testing.T) {
}
bkey, _ := delivery.BuilderBuildKey(name, fp)
b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
gkey, _ := delivery.ControllerGrantKey(name, fp)
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
gkey, _ := delivery.PartGrantKey(name, fp)
grant, _ := delivery.EncodeGrant(delivery.Grant{})
b.putJSON(t, delivery.PartGrantsBucket, gkey, grant)
b.compose(t, "ace", 'c')
b.place(t, "ace", name)
@@ -672,8 +671,8 @@ func (b *bed) fixture(t *testing.T, name, fp, manifest string) {
built, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: digest})
bkey, _ := delivery.BuilderBuildKey(name, fp)
b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
gkey, _ := delivery.ControllerGrantKey(name, fp)
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
gkey, _ := delivery.PartGrantKey(name, fp)
grant, _ := delivery.EncodeGrant(delivery.Grant{})
b.putJSON(t, delivery.PartGrantsBucket, gkey, grant)
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
b.putJSON(t, delivery.PresenceBucket, "ace", on)
@@ -762,7 +761,7 @@ func TestAReleaseAnswerWritesTheWantAgain(t *testing.T) {
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
cleared := false
for k, c := range b.conditionsSaid(t) {
cleared = cleared || (c.Subject.ID == "web" && c.State == asks.StateCleared && strings.Contains(k, ".held."))
cleared = cleared || (strings.HasPrefix(k, "module.web.") && c.State == asks.StateCleared && strings.HasSuffix(k, ".held"))
}
if cleared {
return
@@ -852,7 +851,7 @@ func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) {
continue
}
for k, c := range b.conditionsSaid(t) {
if c.Subject.ID == "web" && c.State == asks.StateCleared && strings.Contains(k, ".held.") && strings.Contains(c.Resolved, "lifted its hold by itself") {
if strings.HasPrefix(k, "module.web.") && c.State == asks.StateCleared && strings.HasSuffix(k, ".held") {
if w := b.wantNow(t, "ace", "web", "the want at the lifted fingerprint", func(w delivery.Want) bool { return w.FP == fNew }); w.Rule != 0 {
t.Errorf("the want = %+v", w)
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go
+6 -53
View File
@@ -66,7 +66,6 @@ var readBuckets = []string{
delivery.PresenceBucket,
delivery.HaveBucket,
delivery.StepsBucket(goBuildModule),
takeoverBucket,
provisions.ProvisionsBucket,
keptBucket,
}
@@ -75,10 +74,6 @@ var readBuckets = []string{
// newest done trunk build of a module names the commit a reader of the manifests runs at (novox/hq ADR 0571 clause 1).
var readLogs = []string{delivery.BuildLogRead}
// takeoverBucket is the condition keeper's take-over key's bucket (novox/hq ADR 0564 §9): whether the assigner's said
// conditions go straight to the router or only as findings (issue 1952 WM6).
var takeoverBucket = health.Bucket(health.KeeperModule, health.TakeoverState)
// goBuildModule is the Go build module, which takes the assigner's compose asks on its seat's check stream and writes
// their steps in its steps state (novox/hq ADR 0482 clause 4).
const goBuildModule = "mesh-build-go"
@@ -157,9 +152,9 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err)
continue
}
if (name == assign.FindingsBucket || name == takeoverBucket) && err != nil {
// New with health leaving the controller (issue 1952 WM6): without them the conditions are said straight to the
// router and kept as no finding, as before.
if name == assign.FindingsBucket && err != nil {
// New with health leaving the controller (issue 1952 WM6): without it the conditions are said straight to the
// router and kept as no finding.
logf("the bucket %s cannot be opened (%v): the assigner's conditions are said straight to the router until it starts again", name, err)
continue
}
@@ -180,7 +175,7 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
// states and those new with a delivery of their own, bound once made, and the two that choose the grants' source.
func startsWithout(name string) bool {
switch name {
case delivery.SeatHoldersBucket, keptBucket, provisions.ProvisionsBucket, assign.PoliciesBucket, assign.FindingsBucket, takeoverBucket,
case delivery.SeatHoldersBucket, keptBucket, provisions.ProvisionsBucket, assign.PoliciesBucket, assign.FindingsBucket,
delivery.AssignerPutsBucket:
return true
}
@@ -222,10 +217,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return follow(ctx, js, conn, kvs, findings.Say(sayVia(ctx, js)), findings, emitting{said}, buildLog, logger)
}
// findingsOf is where the assigner's said conditions go as findings (issue 1952 WM6): its findings bucket, and the
// condition keeper's take-over key, watched, so a condition is said straight to the router only until the take-over. A
// bucket it cannot open is said: without its findings the conditions are said straight, as before; without the take-over
// key they are said straight too, the controller being followed.
// findingsOf is where the assigner's said conditions go as findings (issue 1952 WM6): its findings bucket, which the
// condition keeper carries (novox/hq ADR 0590 §3). Without it the conditions are said straight to the router.
func findingsOf(ctx context.Context, kvs buckets, logf func(string, ...any)) assign.Findings {
var f assign.Findings
if kv := kvs[assign.FindingsBucket]; kv != nil {
@@ -239,46 +232,6 @@ func findingsOf(ctx context.Context, kvs buckets, logf func(string, ...any)) ass
return keys, err
}
}
var keeper atomic.Bool
f.ToKeeper = keeper.Load
takeover := takeoverBucket
kv := kvs[takeover]
if kv == nil {
return f
}
w, err := kv.Watch(ctx, health.TakeoverKey)
if err != nil {
process.NotWatched(logf, takeover, err, "the controller is followed: the conditions are said straight to the router")
return f
}
go func() {
defer w.Stop()
for {
select {
case <-ctx.Done():
return
case e, ok := <-w.Updates():
if !ok {
logf("the watch of %s ended: the last source it named is followed until the assigner starts again", takeover)
return
}
if e == nil {
continue
}
var raw []byte
if e.Operation() == jetstream.KeyValuePut {
raw = e.Value()
}
t, err := health.DecodeTakeover(raw, raw != nil)
if err != nil {
logf("the take-over key cannot be read (%v): the controller is followed", err)
}
if was := keeper.Swap(t.Follow == health.FollowKeeper); was != keeper.Load() {
logf("the assigner's conditions are said by %s from now (the take-over key follows %s)", t.Seat(), t.Follow)
}
}
}
}()
return f
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go
+5 -57
View File
@@ -30,10 +30,8 @@ type passer struct {
// pass says a condition once and a restart says it again (the router keeps one per key).
say func(asks.Condition) error
said map[string]bool
// findings is the assigner's findings bucket (its said conditions and policy-split, ADR 0564 §2, ADR 0565 §8); splits the
// modules whose policy-split finding stands, nil until the first pass, which deletes any a previous process left.
// findings is the assigner's findings bucket (its said conditions, ADR 0564 §2).
findings assign.Findings
splits map[string]bool
// failedProvisions are the failed provisions whose finding this process wrote (ADR 0570 §4).
failedProvisions map[string]bool
// missing are the required provisions with no key this process pulsed, and missingSaid those whose finding it wrote.
@@ -138,7 +136,6 @@ func (p *passer) wants(ctx context.Context) {
defer func() { p.provisionsMissing(ctx, missing) }()
defer p.unplacedFindings()
defer p.seatHolders(ctx)
defer p.policySplits(manifests, chosen)
p.clearStaleCalls()
placed := p.w.placedNodes()
puts := p.putsByModule()
@@ -159,7 +156,7 @@ func (p *passer) wants(ctx context.Context) {
stale := map[string]neitherOne{}
left := map[string]string{}
defer func() { p.sayNeither(neither, stale, left) }()
// Where grants are read from, once for the pass: the grants part's while the lease is handed away (issue 774).
// Where grants are read from, once for the pass: the grants part's (issue 774, ADR 0571 clause 9).
grants := p.w.grantsBucket()
for name := range names {
fp := trunk[name].FP2
@@ -170,7 +167,7 @@ func (p *passer) wants(ctx context.Context) {
return p.w.composed(node, name, fp) // a placed node's own: the digest its own have names (ADR 0571 §6)
},
GrantedAt: func(fp string) bool {
key, err := delivery.ControllerGrantKey(name, fp)
key, err := delivery.PartGrantKey(name, fp)
return err == nil && p.w.has(grants, key)
},
Restored: func(node string) bool {
@@ -183,13 +180,13 @@ func (p *passer) wants(ctx context.Context) {
}}
if fp != "" {
held, _ := delivery.HeldKey(name, fp)
grant, _ := delivery.ControllerGrantKey(name, fp)
grant, _ := delivery.PartGrantKey(name, fp)
hold, _ := delivery.HoldAskKey(name, fp)
m.Released = p.answeredWith(hold, "release")
m.Kept = p.answeredWith(hold, "keep")
m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(delivery.AssignerWantsBucket, held) && !m.Released, p.w.has(grants, grant)
if raw := p.w.value(grants, grant); raw != nil {
if g, err := delivery.DecodeControllerGrant(raw); err == nil {
if g, err := delivery.DecodeGrant(raw); err == nil {
m.GrantedOn = func(node string) bool { return assign.GrantNamesNode(g, node, name) }
}
}
@@ -925,55 +922,6 @@ func chosenOf(chosen map[string]catalogue.PersonPolicy, module string) *catalogu
return nil
}
// policySplits writes the finding policy-split for each module whose policy at the assigner differs from one the controller
// writes beside a composition of it, and deletes it once they agree (ADR 0565 §8): written when a split begins, never again
// while it stands. The first pass after a start deletes every one a previous process left that no longer splits.
func (p *passer) policySplits(manifests map[string]catalogue.Manifest, chosen map[string]catalogue.PersonPolicy) {
if p.findings.Put == nil {
return
}
now := time.Now
if p.now != nil {
now = p.now
}
split := map[string]bool{}
// The controller writes no policy any more (ADR 0571 §6): nothing is compared with.
for _, s := range assign.Splits(manifests, chosen, nil) {
split[s.Module] = true
if p.splits[s.Module] {
continue
}
value, err := assign.PolicySplitFinding(s, now())
if err == nil {
err = p.findings.Put(assign.PolicySplitKey(s.Module), value)
}
if err != nil {
p.logf("the policy split of %s is not written: %v", s.Module, err)
delete(split, s.Module)
}
}
gone := p.splits
if gone == nil && p.findings.Keys != nil {
gone = map[string]bool{}
if keys, err := p.findings.Keys(); err == nil {
for _, k := range keys {
if m, ok := strings.CutSuffix(strings.TrimPrefix(k, "module."), "."+assign.KindPolicySplit); ok && strings.HasPrefix(k, "module.") {
gone[m] = true
}
}
}
}
for m := range gone {
if !split[m] {
if err := p.findings.Delete(assign.PolicySplitKey(m)); err != nil {
p.logf("the policy split of %s is not deleted: %v", m, err)
split[m] = true // tried again at the next pass
}
}
}
p.splits = split
}
// provisionsOf answers, per module, the provisions it consumes on a node, by key (ADR 0570 §4), from the seat
// mesh-provisioner's state; nil while the state is not read (the seat not delivered yet), so no want waits on it. A failed
// provision is also said as the assigner's finding `provider.<key>.provision-failed` (a waiting one is the watchman's
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go
+2 -2
View File
@@ -293,8 +293,8 @@ func putOfTheBus(t *testing.T) {
default:
}
composes(t, p.w, "novox", digest)
gkey, _ := delivery.ControllerGrantKey("nats", fp)
g, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
gkey, _ := delivery.PartGrantKey("nats", fp)
g, _ := delivery.EncodeGrant(delivery.Grant{})
world(delivery.PartGrantsBucket, gkey, g)
got := step()
akey, _ := delivery.RecordAskKey("novox", "nats", digest)
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go
+2 -2
View File
@@ -226,9 +226,9 @@ func (p *passer) composedFor(node, module, fp string) (*delivery.ComposedDigest,
out = &d
}
granted := false
if gkey, err := delivery.ControllerGrantKey(module, fp); err == nil {
if gkey, err := delivery.PartGrantKey(module, fp); err == nil {
if raw := p.w.value(p.w.grantsBucket(), gkey); raw != nil {
if g, err := delivery.DecodeControllerGrant(raw); err == nil {
if g, err := delivery.DecodeGrant(raw); err == nil {
granted = g.NamesNode(node, module)
}
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go
-13
View File
@@ -312,17 +312,6 @@ func (s *seatTools) doAgain(args map[string]any) (any, error) {
return map[string]any{"proposal": p.ID, "again": p.Again, "follow": followProposal(p.ID)}, nil
}
// doAdopt is the verb adopt (§7): gone since the lease write. The seed was the controller's export proposed whole, and the
// mesh has no controller export to take over (ADR 0547 §11, ADR 0567 §11); nothing is written.
func (s *seatTools) doAdopt(args map[string]any, by string) (any, error) {
return nil, errors.New("adopt is gone: placement is the mesh's alone since the lease write, and there is no controller to take it over from (ADR 0547 §11, ADR 0567 §11)")
}
// doHandBack is the verb hand-back (§7): gone since the lease write, as adopt is; nothing is deleted.
func (s *seatTools) doHandBack(args map[string]any, by string) (any, error) {
return nil, errors.New("hand-back is gone: placement is the mesh's alone since the lease write, and there is no controller to hand it back to (ADR 0547 §11, ADR 0567 §11)")
}
// placementAnswer is what the verb placement answers.
type placementAnswer struct {
Keys map[string]keyed `json:"keys,omitempty"`
@@ -496,8 +485,6 @@ func (s *seatTools) tools() []stdio.Tool {
catalogue.PlacementVerbPropose: func(a map[string]any) (any, error) { return s.doPropose(a, stdio.CallerOf(a)) },
catalogue.PlacementVerbWithdraw: func(a map[string]any) (any, error) { return s.doWithdraw(a, stdio.CallerOf(a)) },
catalogue.PlacementVerbAgain: s.doAgain,
catalogue.PlacementVerbAdopt: func(a map[string]any) (any, error) { return s.doAdopt(a, stdio.CallerOf(a)) },
catalogue.PlacementVerbHandBack: func(a map[string]any) (any, error) { return s.doHandBack(a, stdio.CallerOf(a)) },
}
var out []stdio.Tool
for _, v := range seat.Serves {
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go
+9 -49
View File
@@ -35,9 +35,8 @@ func (s memStore) history(state, key string) ([]stdio.StateRevision, error) {
return []stdio.StateRevision{{Key: key, Value: v, Revision: uint64(len(v)), Op: "put"}}, nil
}
// ADR 0567 §2, §7, §10: the seat's seven verbs over the states. propose lays a proposal and writes it with its candidates,
// or refuses naming every finding and writes nothing; withdraw and again mark it for the holder; adopt and hand-back are
// answered as gone and change nothing (ADR 0547 §11, ADR 0567 §11); placement and
// ADR 0567 §2, §7, §10: the seat's five verbs over the states. propose lays a proposal and writes it with its candidates,
// or refuses naming every finding and writes nothing; withdraw and again mark it for the holder; placement and
// proposals read every state the seat keeps, and history names the stream for what the state no longer keeps.
func TestTheSeatsVerbs(t *testing.T) {
anchor := delivery.Placement{V: delivery.Version, Node: "anchor", Modules: []delivery.PlacedModule{{Module: "web"}}}
@@ -59,21 +58,8 @@ func TestTheSeatsVerbs(t *testing.T) {
})
s.now = func() time.Time { return time.Date(2026, 10, 28, 20, 0, 0, 0, time.UTC) }
// 774/S28/AC1: adopt and hand-back are answered as gone, naming their records, and change nothing and ask nobody.
raw, _ := delivery.EncodePlacement(anchor)
st[delivery.PlacementState].values["placement.anchor"] = raw
for verb, call := range map[string]func() (any, error){
"adopt": func() (any, error) { return s.doAdopt(map[string]any{"why": "take over"}, "operator@laptop") },
"hand-back": func() (any, error) { return s.doHandBack(map[string]any{"why": "back"}, "op") },
} {
if _, err := call(); err == nil || !strings.Contains(err.Error(), verb+" is gone") || !strings.Contains(err.Error(), "ADR 0567 §11") {
t.Errorf("%s: %v", verb, err)
}
}
if len(asked) != 0 || len(st[delivery.PlacementProposalsState].values) != 0 || len(st[delivery.PlacementCandidatesState].values) != 0 ||
len(st[delivery.PlacementState].values) != 1 {
t.Fatalf("adopt or hand-back changed something: asked %v", asked)
}
// withdraw and again mark the proposal; the holder ends or judges it.
got, err := s.doPropose(map[string]any{"why": "hold", "changes": `[{"op":"hold","node":"anchor","module":"web","claim":"node-shell","scope":"node"}]`}, "op")
@@ -194,10 +180,8 @@ func TestTheSeatsVerbs(t *testing.T) {
} else if rm := got.(proposed).Proposal; !slices.Equal(rm.Nodes(), []string{"joiner"}) || !slices.Equal(rm.RemovedNodes(), []string{"joiner"}) {
t.Errorf("the remove-node proposed %+v", rm)
}
if _, err := s.doHandBack(map[string]any{"why": "back"}, "op"); err == nil || !strings.Contains(err.Error(), "since the lease write") {
t.Errorf("a hand-back: %v", err)
}
if tools := s.tools(); len(tools) != 7 || tools[2].Name != "mesh-placement.propose" || tools[2].Run == nil || tools[2].Input == nil {
// 1126/B8: the seat serves its five verbs; adopt and hand-back went with the controller (ADR 0590).
if tools := s.tools(); len(tools) != 5 || tools[2].Name != "mesh-placement.propose" || tools[2].Run == nil || tools[2].Input == nil {
t.Errorf("the seat's verbs as served: %+v", tools)
}
t.Run("1953/D4-D6: the seat mesh-wants' upgrade and bus", wantsVerbs)
@@ -379,8 +363,7 @@ func TestTheSeatsVerbs(t *testing.T) {
// wantsVerbs (novox/hq ADR 0565 §7, §8, §9): upgrade answers each module's policy and its source; a person's choice is
// written with who and why, recorded as a hand act in the seat's log and said as policy-changed, and default takes it back;
// the bus is refused; compare lists the modules the controller writes another policy for; the process writes policy-split
// for a split and deletes it once they agree; bus names the node behind the indexer's newest.
// the bus is refused; compare lists each node's composition and the findings; bus names the node behind the indexer's newest.
func wantsVerbs(t *testing.T) {
fpOld, fpNew := delivery.SourcePrefix2+strings.Repeat("1", 64), delivery.SourcePrefix2+strings.Repeat("2", 64)
st := memStore{}
@@ -419,9 +402,10 @@ func wantsVerbs(t *testing.T) {
if p := policyOf("nats"); p.Policy != catalogue.PolicyRecord || p.From != catalogue.FromBus {
t.Errorf("nats = %+v", p)
}
// compare: the controller writes no policy, so no module splits; each node's own composition is the digest its have names.
// compare: each node's own composition is the digest its have names, and the assigner's findings; nothing splits, the
// controller being gone (1126/B8).
out, _ := w.doUpgrade(map[string]any{"compare": "true"}, "op")
if got := out.(map[string]any); len(got["splits"].([]assign.Split)) != 0 || got["empty"] != true || got["findings"] == nil ||
if got := out.(map[string]any); got["splits"] != nil || got["findings"] == nil ||
got["composed"].(map[string]string)["anchor"] != "sha256:"+strings.Repeat("c", 64) || len(got["composed"].(map[string]string)) != 1 {
t.Errorf("compare = %+v", got)
}
@@ -466,27 +450,7 @@ func wantsVerbs(t *testing.T) {
if watches[policiesState] != 0 {
t.Errorf("the seat's own small policies state was watched: %v", watches)
}
// The process says the split as its finding, and deletes it once they agree.
findings := map[string][]byte{}
p := &passer{w: newWorld(), logf: func(string, ...any) {}, findings: assign.Findings{
Put: func(k string, v []byte) error { findings[k] = v; return nil },
Delete: func(k string) error { delete(findings, k); return nil },
Keys: func() ([]string, error) { return []string{"module.gone.policy-split"}, nil }}}
man := map[string]catalogue.Manifest{}
for _, name := range []string{"web", "photos"} {
raw, _, _, _ := st.get(modulesState, name)
mod, _ := delivery.DecodeModule(raw)
mraw, _, _, _ := st.get(manifestsState, mod.Manifest)
kept, _ := delivery.DecodeManifest(mraw)
man[name], _ = catalogue.Decode([]byte(kept.JSON))
}
findings["module.gone.policy-split"] = []byte("{}")
// The controller writes no policy now (ADR 0571 §6): the process compares with nothing, and deletes a finding left.
p.policySplits(man, nil)
if len(findings) != 0 {
t.Errorf("findings = %v, want a stale one deleted", findings)
}
// a person's record for web: written, recorded, said; the split is gone.
// a person's record for web: written, recorded, said.
if _, err := w.doUpgrade(map[string]any{"module": "web", "policy": "record", "why": "the operator checks it"}, "person.jochen"); err != nil {
t.Fatal(err)
}
@@ -495,10 +459,6 @@ func wantsVerbs(t *testing.T) {
len(said) != 1 || !strings.HasPrefix(said[0], "policy-changed policy-changed.web.") {
t.Errorf("chosen %+v, asked %v, said %v", c, asked, said)
}
p.policySplits(man, chosen)
if len(findings) != 0 {
t.Errorf("findings after the choice = %v", findings)
}
if _, err := w.doUpgrade(map[string]any{"module": "nats", "policy": "roll", "why": "x"}, "op"); err == nil {
t.Error("the bus was given a policy")
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/wants_tools.go
+3 -7
View File
@@ -133,7 +133,7 @@ type policyRow struct {
}
// doUpgrade is the verb upgrade: read every module's policy or one's; with policy and why set a person's choice, or take it
// back with default; with compare list the splits.
// back with default; with compare each node's composition and the assigner's findings.
func (s *wantsTools) doUpgrade(args map[string]any, by string) (any, error) {
manifests, _, err := s.manifests()
if err != nil {
@@ -157,12 +157,8 @@ func (s *wantsTools) doUpgrade(args map[string]any, by string) (any, error) {
}
}
}
splits := assign.Splits(manifests, chosen, nil) // the controller writes no policy to differ from (ADR 0571 §6)
if splits == nil {
splits = []assign.Split{}
}
out := map[string]any{"splits": splits, "empty": len(splits) == 0, "composed": composed}
// The assigner's findings, its said conditions and its policy splits as the condition keeper reads them (§8, §15).
out := map[string]any{"composed": composed}
// The assigner's findings, its said conditions as the condition keeper reads them (§15).
if found, err := s.values(health.FindingsState); err != nil {
out["findings-unread"] = err.Error()
} else {
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world_test.go
+8 -20
View File
@@ -8,7 +8,6 @@ import (
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-assigner/internal/assign"
@@ -128,33 +127,22 @@ func theWorldCountsTheValuesAKeyTakes(t *testing.T) {
if got := w.changed(delivery.HaveBucket, "gone"); got != 0 {
t.Errorf("changed = %d for a key never seen", got)
}
// ADR 0571 clause 9: grants are read from the grants part's bucket in every case, a lease held, handed away, absent or
// unreadable alike, and the controller's own grants count for nothing.
// ADR 0571 clause 9: grants are read from the grants part's bucket, the one bucket of grants (ADR 0590).
fp := delivery.SourcePrefix2 + strings.Repeat("d", 64)
gkey, _ := delivery.ControllerGrantKey("web", fp)
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{Accounts: []string{"ace.web"}})
gkey, _ := delivery.PartGrantKey("web", fp)
grant, _ := delivery.EncodeGrant(delivery.Grant{Accounts: []string{"ace.web"}})
for _, row := range []struct {
name, lease string
inPart, inController bool
granted bool
name string
inPart bool
granted bool
}{
{"handed away, the part's grant", `{"handed-to":"the operator's hand-over"}`, true, false, true},
{"handed away, only the controller's grant", `{"handed-to":"x"}`, false, true, false},
{"a controller holds it, the part's grant", `{"instance":"novox/1","epoch":3}`, true, false, true},
{"a controller holds it, only its own grant", `{"instance":"novox/1"}`, false, true, false},
{"no lease read", ``, true, false, true},
{"an unreadable lease", `[1]`, true, false, true},
{"the part's grant", true, true},
{"no grant", false, false},
} {
p := newPasser(buckets{}, time.Now())
if row.lease != "" {
p.w.put(buscompose.LeaseBucket, buscompose.ControllerLeaseKey, []byte(row.lease))
}
if row.inPart {
p.w.put(delivery.PartGrantsBucket, gkey, grant)
}
if row.inController {
p.w.put(delivery.ControllerGrantsBucket, gkey, grant)
}
if got := p.w.grantsBucket(); got != delivery.PartGrantsBucket {
t.Errorf("%s: grants are read from %s, want the part's", row.name, got)
}
modules/mesh-delivery-assigner/go.mod
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-assigner
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
modules/mesh-delivery-assigner/go.sum
+2 -6
View File
@@ -1,9 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101085317-790d1c82d027 h1:a33dsavoaqllh40eVICbhxi6Kero1EGSjt1V+2fa2Hk=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101085317-790d1c82d027/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c h1:/YCxh+NYfqry3PxZYY1nQXaccupobxpsFt8c9BYuGBg=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21 h1:4I4ayo0fZYP4PBfEX1sGDDd6arnw5IQJySJ3jcFobPM=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791 h1:7SYIDPMxbPBr8dMB6k9fYsgVt4ZyWVL41q4LMFsfAvs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
modules/mesh-delivery-assigner/internal/assign/asks_test.go
+11 -9
View File
@@ -78,7 +78,7 @@ func TestTheRestoreAskIsValid(t *testing.T) {
func TestTheAssignersConditions(t *testing.T) {
t.Run("the held condition names module, node and words", heldConditionNamesModuleNodeAndWords)
t.Run("a lifted hold clears its condition saying why", liftedHoldClearsItsConditionSayingWhy)
t.Run("1952/WM6: each said condition is a finding, its clear deletes it, said straight only until the take-over", saidConditionsAreFindings)
t.Run("1952/WM6: each said condition is a finding, its clear deletes it, said straight only with no findings bucket", saidConditionsAreFindings)
}
func heldConditionNamesModuleNodeAndWords(t *testing.T) {
@@ -106,8 +106,9 @@ func liftedHoldClearsItsConditionSayingWhy(t *testing.T) {
}
// 1952/WM6 (ADR 0564 §2, issue 1952 note 16): every condition the assigner says is a finding in its own bucket, under
// `<scope>.<id>.<kind>`, written at the raise and deleted at the clear; it is said straight to the router while the take-over
// key follows the controller, and only as a finding from the take-over.
// `<scope>.<id>.<kind>`, written at the raise and deleted at the clear, and only as a finding: the keeper is the one
// source since the controller's end (novox/hq ADR 0590 §3, issue 1126 B8). Said straight to the router only where the
// findings bucket cannot be written to at all.
func saidConditionsAreFindings(t *testing.T) {
at := time.Date(2026, 10, 29, 21, 0, 0, 0, time.UTC)
for _, c := range []struct {
@@ -121,24 +122,25 @@ func saidConditionsAreFindings(t *testing.T) {
{BackupCondition("ace", "mail", fNew, "disk full", true), BackupCondition("ace", "mail", fNew, "", false), "module.ace.mail." + strings.TrimPrefix(fNew, delivery.SourcePrefix2) + ".backup"},
} {
puts, deletes, said := map[string]health.Finding{}, []string{}, 0
keeper := false
f := Findings{Put: func(k string, v []byte) error {
got, err := health.DecodeFinding(v)
puts[k] = got
return err
}, Delete: func(k string) error { deletes = append(deletes, k); return nil }, ToKeeper: func() bool { return keeper }, Now: func() time.Time { return at }}
}, Delete: func(k string) error { deletes = append(deletes, k); return nil }, Now: func() time.Time { return at }}
say := f.Say(func(asks.Condition) error { said++; return nil })
if err := say(c.raised); err != nil {
t.Fatalf("%s: %v", c.key, err)
}
got, ok := puts[c.key]
if !ok || got.Severity != health.Severity(c.raised.Severity) || got.Headline != c.raised.Headline || got.Observer != delivery.AssignerModule ||
got.Subject.Machine != c.raised.Subject.Machine || said != 1 {
got.Subject.Machine != c.raised.Subject.Machine || said != 0 {
t.Fatalf("%s: finding %+v (written %v), said %d", c.key, got, puts, said)
}
keeper = true
if err := say(c.cleared); err != nil || !slices.Equal(deletes, []string{c.key}) || said != 1 {
t.Fatalf("%s: deleted %v, said %d after the take-over (%v)", c.key, deletes, said, err)
if err := say(c.cleared); err != nil || !slices.Equal(deletes, []string{c.key}) || said != 0 {
t.Fatalf("%s: deleted %v, said %d (%v)", c.key, deletes, said, err)
}
if err := (Findings{}).Say(func(asks.Condition) error { said++; return nil })(c.raised); err != nil || said != 1 {
t.Fatalf("%s: with no findings bucket said %d (%v), want straight once", c.key, said, err)
}
}
if _, _, _, err := FindingOf(asks.Condition{Key: "another.x", State: asks.StateRaised}, at); err == nil {
modules/mesh-delivery-assigner/internal/assign/composer.go
+3 -3
View File
@@ -23,9 +23,9 @@ const (
// composer is built at its repository's root, at the mesh-sdk its go.mod pins.
const SDKDir = "go"
// The modules whose mesh-sdk reads the manifests a plan sends (ADR 0482 clause 5): the controller composes and sends
// them, the node-engine applies them. Each is judged at its repository.
var ReaderModules = map[string]string{"mesh-controller": "novox/mesh-controller", "mesh-host": "novox/mesh-host"}
// The modules whose mesh-sdk reads the manifests a plan sends (ADR 0482 clause 5): the node-engine composes and applies
// them, the controller being gone (novox/hq ADR 0590, issue 1126). Each is judged at its repository.
var ReaderModules = map[string]string{"mesh-host": "novox/mesh-host"}
// HeadJob is the compose job of a group with a mesh-sdk member (ADR 0482 clauses 3 and 4, issue 1622): the composer
// built at the member's head composes the machines the change touches, as the trunk holds them and as the change leaves
modules/mesh-delivery-assigner/internal/assign/findings.go
+9 -10
View File
@@ -59,18 +59,17 @@ func FindingOf(c asks.Condition, at time.Time) (key string, value []byte, raised
return key, value, true, err
}
// Findings is where the assigner's findings go: the bucket's put and delete, and whether the keeper's conditions are the
// ones said (the take-over key names the keeper).
// Findings is where the assigner's findings go: the bucket's put and delete.
type Findings struct {
Put func(key string, value []byte) error
Delete func(key string) error
Keys func() ([]string, error)
ToKeeper func() bool
Now func() time.Time
Put func(key string, value []byte) error
Delete func(key string) error
Keys func() ([]string, error)
Now func() time.Time
}
// Say is say with each condition also written as a finding (or its finding deleted), and said straight to the router only
// while the take-over key follows the controller. A finding that cannot be written is the error, beside the say's.
// Say is say with each condition written as a finding (or its finding deleted), which the condition keeper carries, the
// one source since the controller's end (novox/hq ADR 0590 §3); said straight to the router only where there is no
// findings bucket at all. A finding that cannot be written is the error.
func (f Findings) Say(say func(asks.Condition) error) func(asks.Condition) error {
return func(c asks.Condition) error {
var errs []error
@@ -93,7 +92,7 @@ func (f Findings) Say(say func(asks.Condition) error) func(asks.Condition) error
}
}
}
if f.ToKeeper == nil || !f.ToKeeper() {
if f.Put == nil {
if say != nil {
if err := say(c); err != nil {
errs = append(errs, err)
modules/mesh-delivery-assigner/internal/assign/policy.go
+7 -60
View File
@@ -17,8 +17,6 @@ import (
// A person's upgrade policy is the assigner's (novox/hq ADR 0565 §7, §8, issue 1953 D4, D5): kept in its state `policies`, one
// key per module, set and taken back through `mesh-wants.upgrade`. Every reader takes one derivation, mesh-sdk's
// catalogue.Upgrade: a person's choice over the module's word, its irreplaceable data and the default, the bus never rolled.
// Until the lease write the controller still writes a policy beside each composition's digest; where it differs from the
// assigner's, the assigner says so as its finding `policy-split`, naming mesh-wants.upgrade as the remedy.
// PoliciesBucket is the assigner's state `policies`.
var PoliciesBucket = delivery.AssignerModule + "_" + delivery.AssignerPoliciesState
@@ -57,64 +55,6 @@ func contains(list []string, s string) bool {
return false
}
// Split is a module whose policy at the assigner differs from one the controller writes beside a composition of it.
type Split struct {
Module string `json:"module"`
Policy string `json:"policy"`
From string `json:"from"`
Controller []string `json:"controller"`
}
// Splits is every module of manifests the controller writes a policy for that differs from the assigner's, sorted.
func Splits(manifests map[string]catalogue.Manifest, chosen map[string]catalogue.PersonPolicy, controller map[string][]string) []Split {
var out []Split
for module, theirs := range controller {
m, ok := manifests[module]
if !ok {
continue
}
policy, from, _, _ := PolicyOf(m, chosen)
for _, p := range theirs {
if p != policy {
out = append(out, Split{Module: module, Policy: policy, From: from, Controller: theirs})
break
}
}
}
sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module })
return out
}
// KindPolicySplit is the assigner's finding of a split policy (ADR 0565 §8).
const KindPolicySplit = "policy-split"
// PolicySplitKey is a module's policy-split finding's key.
func PolicySplitKey(module string) string {
return health.FindingKey(health.ScopeModule, module, KindPolicySplit)
}
// PolicySplitFinding is the finding of one split, in mesh-sdk's layout.
func PolicySplitFinding(s Split, at time.Time) ([]byte, error) {
summary := fmt.Sprintf("%s's upgrade policy is %s at the assigner (%s) and %s at the controller", s.Module, s.Policy, s.From,
strings.Join(s.Controller, " and "))
return health.EncodeFinding(health.Finding{Key: PolicySplitKey(s.Module), Kind: KindPolicySplit,
Subject: health.Subject{Scope: health.ScopeModule, ID: s.Module}, Severity: health.Warning, Summary: summary,
Headline: clipHeadline(s.Module + "'s upgrade policy is split"),
Needs: "set it with mesh-wants.upgrade, so the assigner and the controller agree",
Explanation: asks.NeedsYou + "set it with mesh-wants.upgrade, so the assigner and the controller agree." + " The controller still " +
"writes a policy beside each composition until it hands its lease away; a choice made at its upgrade verb after the copy " +
"is not the assigner's.",
Resolved: s.Module + "'s upgrade policy agrees again", Evidence: []health.Evidence{{At: at.UTC(), Said: summary}},
Source: "assigner." + KindPolicySplit, Observer: delivery.AssignerModule, At: at.UTC()})
}
func clipHeadline(s string) string {
if len([]rune(s)) <= 60 {
return s
}
return string([]rune(s)[:60])
}
// KindProvisionFailed is the assigner's finding of a provision its module's want waits for that failed (ADR 0570 §4).
const KindProvisionFailed = "provision-failed"
@@ -186,3 +126,10 @@ func UnplacedFinding(node string, at time.Time) ([]byte, error) {
Resolved: node + " is placed", Evidence: []health.Evidence{{At: at.UTC(), Said: summary}},
Source: "assigner." + KindUnplaced, Observer: delivery.AssignerModule, At: at.UTC()})
}
func clipHeadline(s string) string {
if len([]rune(s)) <= 60 {
return s
}
return string([]rune(s)[:60])
}
modules/mesh-delivery-assigner/internal/assign/wants.go
+2 -2
View File
@@ -733,8 +733,8 @@ func (m Module) userListRevision(node string, before delivery.Want, had bool, n
return w, true
}
// GrantNamesNode says whether a grant of module names its accounts on node (mesh-sdk's ControllerGrant.NamesNode, which a
// GrantNamesNode says whether a grant of module names its accounts on node (mesh-sdk's Grant.NamesNode, which a
// put reads by too).
func GrantNamesNode(g delivery.ControllerGrant, node, module string) bool {
func GrantNamesNode(g delivery.Grant, node, module string) bool {
return g.NamesNode(node, module)
}
modules/mesh-delivery-assigner/internal/assign/wants_test.go
+2 -2
View File
@@ -293,7 +293,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
// Rule 6 per node: a grant written for the module's other nodes does not grant a node whose account it does not name
// (issue 1945, live on 2026-10-25 at 20:04: ace's mesh-build-go was wanted with no account on the bus).
m, n := ready()
grant := delivery.ControllerGrant{Accounts: []string{"ace.mail", "ace.mail.send"}}
grant := delivery.Grant{Accounts: []string{"ace.mail", "ace.mail.send"}}
m.GrantedOn = func(node string) bool { return GrantNamesNode(grant, node, "mail") }
out := Wants(m, n, wantedOld(), nil)
if w := out.Wants["ace"]; w.FP != fNew {
@@ -312,7 +312,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
{[]string{"g14.mailer"}, "g14", "mail", false},
{[]string{"ace.mail"}, "g14", "mail", false},
} {
if got := GrantNamesNode(delivery.ControllerGrant{Accounts: c.accounts}, c.node, c.module); got != c.named {
if got := GrantNamesNode(delivery.Grant{Accounts: c.accounts}, c.node, c.module); got != c.named {
t.Errorf("%v names %s of %s: %v, want %v", c.accounts, c.module, c.node, got, c.named)
}
}
modules/mesh-delivery-assigner/manifest_test.go
+6 -6
View File
@@ -33,7 +33,7 @@ func TestManifest(t *testing.T) {
}
reads := slices.Clone(m.Reads)
slices.Sort(reads)
want := []string{"mesh-build-go.steps", "mesh-conditions.takeover", "mesh-config.kept", "mesh-config.resolved",
want := []string{"mesh-build-go.steps", "mesh-config.kept", "mesh-config.resolved",
"mesh-delivery-builder.build-log", "mesh-delivery-builder.builds", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests",
"mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-grants.grants", "mesh-provisioner.provisions", "node-engine.have"}
if !reflect.DeepEqual(reads, want) {
@@ -60,14 +60,14 @@ func TestManifest(t *testing.T) {
}
// ADR 0546: it serves its own tools (put, release, puts) from its bundle through the tool runner (the artifact loads the
// module), says each put's step as the event put, and calls the hand-act seat and the bus node's backup holder.
// ADR 0567 §1, §6-§10: it holds the seat mesh-placement, serving its seven verbs; keeps the seat's events on its stream
// placement for 30 days; calls the controller's export, its four verbs that keep its assignments in step and its
// hand-act log; and reads mesh-config's resolved values to compose every node a proposal touches.
// ADR 0567 §1, §6-§10: it holds the seat mesh-placement, serving its five verbs (adopt and hand-back went with the
// controller, ADR 0590); keeps the seat's events on its stream placement for 30 days; calls the hand-act log; and reads
// mesh-config's resolved values to compose every node a proposal touches.
// ADR 0565 §7, §9 (issue 1953 D4): and the seat mesh-wants, serving upgrade and bus.
if len(m.Claims) != 2 || m.Claims[0].Name != catalogue.PlacementSeat || m.Claims[0].At() != catalogue.ScopeMesh ||
!slices.Equal(m.Claims[0].Serves, []string{"placement", "proposals", "propose", "withdraw", "again", "adopt", "hand-back"}) ||
!slices.Equal(m.Claims[0].Serves, []string{"placement", "proposals", "propose", "withdraw", "again"}) ||
m.Claims[1].Name != catalogue.WantsSeat || m.Claims[1].At() != catalogue.ScopeMesh || !slices.Equal(m.Claims[1].Serves, []string{"upgrade", "bus"}) {
t.Errorf("the assigner claims %+v, want mesh-placement with its seven verbs and mesh-wants with its two", m.Claims)
t.Errorf("the assigner claims %+v, want mesh-placement with its five verbs and mesh-wants with its two", m.Claims)
}
if seat, _ := catalogue.SeatNamed(catalogue.WantsSeat); catalogue.CanHold(m, seat) != nil {
t.Errorf("the assigner cannot hold mesh-wants: %v", catalogue.CanHold(m, seat))
modules/mesh-delivery-assigner/module.json
+1 -4
View File
@@ -11,9 +11,7 @@
"proposals",
"propose",
"withdraw",
"again",
"adopt",
"hand-back"
"again"
]
},
{
@@ -27,7 +25,6 @@
],
"reads": [
"mesh-build-go.steps",
"mesh-conditions.takeover",
"mesh-config.kept",
"mesh-config.resolved",
"mesh-delivery-builder.build-log",
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go
+2 -2
View File
@@ -188,8 +188,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
defer cc.Stop()
}
// The controller says artifacts-gone when its take-in found a key's blob gone (issue 1937): the builder's durable
// consumer on EVENTS, made by the controller from the manifest's consumes. Where it is not there yet the builder says
// The seat mesh-artifact-sweep says artifacts-gone when it found a key's blob gone (issue 1937, ADR 0571 §2): the
// builder's durable consumer on EVENTS, made by the grants part from the manifest's consumes. Where it is not there yet the builder says
// so; it still asks held at its start and at every module value.
switch cons, err := cred.EventConsumer(ctx, js, eventsStream); {
case err != nil:
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/stored.go
+3 -4
View File
@@ -15,8 +15,8 @@ import (
"mesh-delivery-builder/internal/build"
)
// eventsStream is the stream every module's and role's events are on, where the controller makes the builder's durable
// consumer of artifacts-gone (mesh-controller internal/broker EventsStream).
// eventsStream is the stream every module's and role's events are on, where the grants part makes the builder's durable
// consumer of artifacts-gone (mesh-sdk buscompose EventsStream).
const eventsStream = "EVENTS"
// SeatHeld is the seat mesh-artifact-sweep's verb held (ADR 0571 §2).
@@ -27,8 +27,7 @@ type toolCaller func(key string, body any) (json.RawMessage, error)
// storedVia is build.Builder's Stored: it asks the seat mesh-artifact-sweep's `held` (novox/hq ADR 0571 §2, issue 1937)
// and answers the references the store no longer holds. One it could not ask about, with none gone, is an error: the key
// is then left done, and the controller's take-in still refuses a gone blob. The controller's `held` is not asked: it
// is no longer a fallback (issue 1955).
// is then left done, and asked again at the next module value or artifacts-gone.
func storedVia(call toolCaller) func(ctx context.Context, references []string) ([]string, error) {
return func(_ context.Context, references []string) ([]string, error) {
body := map[string]any{"references": references}
modules/mesh-delivery-builder/go.mod
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-builder
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101143343-67354f8e019a
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
modules/mesh-delivery-builder/go.sum
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101143343-67354f8e019a h1:AChSJUmHyOZ5smiuTrtDZNjetYNTFb+anRFjXIQHbSA=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101143343-67354f8e019a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791 h1:7SYIDPMxbPBr8dMB6k9fYsgVt4ZyWVL41q4LMFsfAvs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
modules/mesh-delivery-builder/internal/build/run.go
+2 -2
View File
@@ -260,8 +260,8 @@ func (b *Builder) storedOnce(key string) func(ctx context.Context, references []
}
}
// ArtifactsGone takes the controller's event artifacts-gone in (issue 1937): a take-in found that the store does not
// hold what the module's key names. The module is looked at again, which asks the store (ClaimOptions.Stored) and
// ArtifactsGone takes the seat mesh-artifact-sweep's event artifacts-gone in (issue 1937, ADR 0571 §2): the sweep found
// that the store does not hold what the module's key names. The module is looked at again, which asks the store (ClaimOptions.Stored) and
// builds the key again; an event for a fingerprint the module no longer has is said and left. The event only wakes the
// builder: what is built rests on the store's answer, never on the event alone.
func (b *Builder) ArtifactsGone(ctx context.Context, e delivery.ArtifactsGone) {
modules/mesh-delivery-builder/manifest_test.go
+3 -3
View File
@@ -112,9 +112,9 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
t.Errorf("invokes = %v, config = %v, want %v and no config", m.Invokes, m.Config, want)
}
// The builder is woken by a build module's report of a toolchain, as the indexer is: the bucket may not exist when it
// starts; and by the controller's artifacts-gone, a take-in refused for a gone blob (1937/AC4).
// 1955/L7: and the seat mesh-artifact-sweep's, the controller's until the lease write (its take-in sweeps until then).
if want := []string{"mesh-artifact-sweep.artifacts-gone", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-controller.artifacts-gone"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
// starts; and by the seat mesh-artifact-sweep's artifacts-gone, a blob the sweep found gone (1937/AC4, 1955/L7), the
// controller's no longer (1126/B9, ADR 0590).
if want := []string{"mesh-artifact-sweep.artifacts-gone", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
t.Errorf("consumes = %v, want %v", m.Consumes, want)
}
if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" {
modules/mesh-delivery-builder/module.json
+1 -2
View File
@@ -16,8 +16,7 @@
"consumes": [
"mesh-artifact-sweep.artifacts-gone",
"mesh-build-go.toolchain.reported",
"mesh-build-node.toolchain.reported",
"mesh-controller.artifacts-gone"
"mesh-build-node.toolchain.reported"
],
"invokes": [
"seat:mesh-artifact-sweep.held"
modules/mesh-delivery-checker/go.mod
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-checker
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101143343-67354f8e019a
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
modules/mesh-delivery-checker/go.sum
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101143343-67354f8e019a h1:AChSJUmHyOZ5smiuTrtDZNjetYNTFb+anRFjXIQHbSA=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101143343-67354f8e019a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791 h1:7SYIDPMxbPBr8dMB6k9fYsgVt4ZyWVL41q4LMFsfAvs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
modules/mesh-delivery-checker/internal/check/beside.go
+4 -8
View File
@@ -2,7 +2,6 @@ package check
import (
"context"
"fmt"
"strconv"
)
@@ -33,12 +32,13 @@ const (
// besideRules are the repositories each core repository's tests read, as the tests state it (mesh-sdk go/internal/beside
// beside.go): the catalogue, the node-engine, the SDK, and the delivery's pinned forge dependencies (mesh-catalog
// checks/deps). Which commit of each is besideCommit's rule.
// checks/deps). Which commit of each is besideCommit's rule. No repository's tests read the controller, which ended by its
// unassignment (novox/hq ADR 0590, issue 1126); its own repository keeps its siblings while it holds a module.
var besideRules = map[string][]string{
controllerRepo: {catalogueRepository, nodeEngineRepo, SDKRepository},
nodeEngineRepo: {catalogueRepository, SDKRepository, controllerRepo},
nodeEngineRepo: {catalogueRepository, SDKRepository},
SDKRepository: {catalogueRepository, nodeEngineRepo},
catalogueRepository: {SDKRepository, controllerRepo},
catalogueRepository: {SDKRepository},
"novox/mesh-delivery": {SDKRepository, catalogueRepository},
}
@@ -79,10 +79,6 @@ func (j *Examiner) besideFor(ctx context.Context, c *Case, p Pull) ([]BesideRef,
out = append(out, BesideRef{Repository: repo, Commit: member.Head, Ref: refOf(member.Number)})
} else if head := heads[repo]; head != "" {
out = append(out, BesideRef{Repository: repo, Commit: head, atTrunk: true})
} else if repo == controllerRepo && p.Repository == nodeEngineRepo {
// The node-engine's step-nine test judges the controller's own manifest and fails without it (issue 1295): a
// checker that cannot place it says so, where placing none left that test to skip.
return nil, fmt.Errorf("the trunk head of %s is not known, so the controller the tests of %s#%d read cannot be placed beside them", repo, p.Repository, p.Number)
}
}
}
modules/mesh-delivery-checker/internal/check/pins_test.go
+26 -40
View File
@@ -2,6 +2,7 @@ package check
import (
"maps"
"slices"
"strings"
"testing"
@@ -116,34 +117,31 @@ func TestTheSiblingsAreWhereTheTestsStateThem(t *testing.T) { // review D2, ADR
}
}
// 1295/R4g: the node-engine's tests read the controller's manifest, so a mesh-host pull request has mesh-controller beside it:
// at the group member's head when the group holds one, else at its trunk.
func hostCase(t *testing.T, pulls ...delivery.CandidatePull) *Case {
t.Helper()
c, err := delivery.DecodeCandidate(candidateOf(t, "feat/x", pulls...))
if err != nil {
t.Fatal(err)
}
cs := CaseOf(c, map[string]string{"novox/mesh-host": trunkA, controllerRepo: trunkA2})
if cs == nil {
t.Fatal("no case")
}
return cs
}
func TestTheNodeEnginesTestsHaveTheControllerBesideThem(t *testing.T) {
// 1126/B8 (novox/hq ADR 0590): the controller is gone, so no repository's tests read it: a mesh-host pull request has
// the catalogue and mesh-sdk beside it and no mesh-controller, even where its trunk is known and the group holds one.
func TestNoRepositorysTestsHaveTheControllerBesideThem(t *testing.T) {
const host = "novox/mesh-host"
b := newBench(t)
b.ex.Trunks = trunksOf(t, map[string]string{host: trunkA, controllerRepo: trunkA2, catalogueRepository: trunkS, SDKRepository: trunkS})
b.judge(t, hostCase(t, pullOf(host, 5, head1)))
if c := besideOf(t, b, host)[controllerRepo]; c.Commit != trunkA2 || c.Ref != "" {
t.Errorf("the controller beside the node-engine alone = %+v, want its trunk %s", c, trunkA2)
c, err := delivery.DecodeCandidate(candidateOf(t, "feat/x", pullOf(host, 5, head1), pullOf(controllerRepo, 9, head2)))
if err != nil {
t.Fatal(err)
}
b = newBench(t)
b.ex.Trunks = trunksOf(t, map[string]string{host: trunkA, controllerRepo: trunkA2, catalogueRepository: trunkS, SDKRepository: trunkS})
b.judge(t, hostCase(t, pullOf(host, 5, head1), pullOf(controllerRepo, 9, head2)))
if c := besideOf(t, b, host)[controllerRepo]; c.Commit != head2 || c.Ref != "refs/pull/9/head" {
t.Errorf("the controller beside the node-engine in a group = %+v, want the member's head %s", c, head2)
if v := b.judge(t, CaseOf(c, map[string]string{host: trunkA, controllerRepo: trunkA2})).Verdict; v.State == delivery.VerdictFail &&
strings.Contains(v.Failed.Words, "mesh-controller") {
t.Fatalf("verdict = %+v", v)
}
got := besideOf(t, b, host)
if _, ok := got[controllerRepo]; ok {
t.Errorf("the controller is beside the node-engine: %+v", got)
}
if _, ok := got[catalogueRepository]; !ok {
t.Errorf("the catalogue is not beside the node-engine: %+v", got)
}
for repo, siblings := range besideRules {
if repo != controllerRepo && slices.Contains(siblings, controllerRepo) {
t.Errorf("the tests of %s read the controller", repo)
}
}
}
@@ -158,15 +156,15 @@ func TestTheChangesOwnRepositoryIsBesideItAtItsTrunk(t *testing.T) {
}
}
// 1887/A12: a catalogue change's siblings hold the catalogue itself at the trunk it is merged onto, beside mesh-sdk and
// mesh-controller, so the catalogue's shrink-only list tests read $MESH_CHECK_BESIDE/mesh-catalog.
// 1887/A12: a catalogue change's siblings hold the catalogue itself at the trunk it is merged onto, beside mesh-sdk, so
// the catalogue's shrink-only list tests read $MESH_CHECK_BESIDE/mesh-catalog.
func TestACatalogueChangeHasTheCatalogueBesideItAtItsTrunk(t *testing.T) {
b := newBench(t)
b.ex.Trunks = trunksOf(t, map[string]string{catalogueRepository: trunkA, SDKRepository: trunkS, controllerRepo: trunkA2})
out := b.judge(t, caseOf(t, pullOf(catalogueRepository, 12, head1)))
// ADR 0548 §6: the outcome names the siblings read at their trunk's head, never the repository it merges.
if out.Beside[SDKRepository] != trunkS || out.Beside[controllerRepo] != trunkA2 || out.Beside[catalogueRepository] != "" {
t.Errorf("the siblings named for the verdict = %v, want mesh-sdk and mesh-controller at their trunks and not the catalogue", out.Beside)
if out.Beside[SDKRepository] != trunkS || out.Beside[controllerRepo] != "" || out.Beside[catalogueRepository] != "" {
t.Errorf("the siblings named for the verdict = %v, want mesh-sdk at its trunk and neither the controller nor the catalogue", out.Beside)
}
got := besideOf(t, b, catalogueRepository)
if c := got[catalogueRepository]; c.Commit != trunkA || c.Ref != "" {
@@ -198,15 +196,3 @@ func TestAMeshLabHeadGetsACheck(t *testing.T) {
t.Fatalf("mesh-lab was not fetched: %+v", fetched)
}
}
// 1295: the node-engine's step-nine test fails without the controller beside it, so a checker that knows no trunk head of the
// controller fails the check at the placing, loudly, and does not place none.
func TestTheNodeEnginesCheckFailsWhereTheControllersTrunkIsNotKnown(t *testing.T) {
const host = "novox/mesh-host"
b := newBench(t)
b.ex.Trunks = trunksOf(t, map[string]string{host: trunkA, catalogueRepository: trunkS, SDKRepository: trunkS})
v := b.judge(t, hostCase(t, pullOf(host, 5, head1))).Verdict
if v.State != delivery.VerdictFail || !strings.Contains(v.Failed.Words, "mesh-controller") || !strings.Contains(v.Failed.Words, "cannot be placed") {
t.Fatalf("verdict = %+v", v)
}
}
modules/mesh-delivery-forge-bridge/go.mod
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-forge-bridge
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
modules/mesh-delivery-forge-bridge/go.sum
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 h1:x8k7IeMirMTxUo8BH+xUO+MDnHYgU+636jl1NIw1wvs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791 h1:7SYIDPMxbPBr8dMB6k9fYsgVt4ZyWVL41q4LMFsfAvs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
modules/mesh-delivery-indexer/go.mod
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-indexer
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
modules/mesh-delivery-indexer/go.sum
+2 -6
View File
@@ -1,9 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101143343-67354f8e019a h1:AChSJUmHyOZ5smiuTrtDZNjetYNTFb+anRFjXIQHbSA=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101143343-67354f8e019a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c h1:/YCxh+NYfqry3PxZYY1nQXaccupobxpsFt8c9BYuGBg=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21 h1:4I4ayo0fZYP4PBfEX1sGDDd6arnw5IQJySJ3jcFobPM=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791 h1:7SYIDPMxbPBr8dMB6k9fYsgVt4ZyWVL41q4LMFsfAvs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
modules/mesh-delivery-indexer/internal/beside/beside.go
+4 -5
View File
@@ -16,11 +16,10 @@
// check gets is chosen from the inventory: mesh-catalog by the source of the `nats` module, mesh-host
// by the source of `mesh-host`, and mesh-sdk as the controller's sibling at the commit the controller's
// go.mod pins. Each repository's tests read these siblings, and the checker places exactly these
// (mesh-delivery's checker, besideRules, follows this list): mesh-controller's read mesh-catalog,
// mesh-host and mesh-sdk; mesh-host's read mesh-catalog, mesh-sdk and mesh-controller (its step-nine
// test reads the controller's own manifest, novox/hq issue 1295); mesh-sdk's read mesh-catalog and
// mesh-host; mesh-catalog's read mesh-sdk and mesh-controller; mesh-delivery's read mesh-sdk and
// mesh-catalog. In a mesh where either module has no source repository nothing is
// (mesh-delivery's checker, besideRules, follows this list): mesh-host's read mesh-catalog and
// mesh-sdk; mesh-sdk's read mesh-catalog and mesh-host; mesh-catalog's read mesh-sdk; mesh-delivery's
// read mesh-sdk and mesh-catalog. None reads mesh-controller, which ended by its unassignment (novox/hq
// ADR 0590). In a mesh where either module has no source repository nothing is
// cloned, and these tests fail loudly with "not beside this check": a cause in the setup, not in the
// change. And in a delivery group that holds a mesh-catalog pull request, these tests read the
// catalogue's main, not the group's head.
modules/mesh-delivery-indexer/internal/index/candidates.go
+1
View File
@@ -84,6 +84,7 @@ type Indexer struct {
toolchains map[string]map[string]delivery.Toolchain // language, then node (toolchains.go)
tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go)
unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread
moving map[string]delivery.Forge // by modules key: a merge naming a module another repository holds, waiting for that trunk to drop it (moves.go)
retrying atomic.Bool // a RetryUnread runs in the background (kickRetry)
retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one
retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on
modules/mesh-delivery-indexer/internal/index/modules.go
+24 -3
View File
@@ -22,7 +22,8 @@ import (
// 1851), so a module added by the merge gets a value, and a fingerprint, with no build anywhere. The value of today's
// form (Module.FP, None) is no longer written: it was computed from those records (ADR 0453 clause 6). A module the
// bucket holds of another repository is left as it is and said: a move between repositories is the operator's (ADR 0420
// clause 7). Two directories naming one module write the one the bucket holds, or the first by name, and say the other.
// clause 7), and kept waiting: once the holding repository's trunk drops the module, the merge that drops it hands it to the
// waiting repository instead of deleting it (moves.go). Two directories naming one module write the one the bucket holds, or the first by name, and say the other.
// A value the module already holds but for its commit is not written again: its commit stays the one its fingerprint first
// appeared at (writeOverlay), so a merge writes only the modules it changes.
//
@@ -36,6 +37,12 @@ func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) (*mergeWri
// deleted. A message that names only some of a repository's modules (a registered module read on its own, registered.go)
// drops none.
func (ix *Indexer) applyManifests(ctx context.Context, f delivery.Forge, drop bool) (*mergeWrite, error) {
return ix.applyMoving(ctx, f, drop, nil)
}
// applyMoving is applyManifests; moves names the modules keys the message may take from the repository that holds them
// (a move handed over, moves.go). Every other module held of another repository is left, said and kept waiting.
func (ix *Indexer) applyMoving(ctx context.Context, f delivery.Forge, drop bool, moves map[string]bool) (*mergeWrite, error) {
w := newMergeWrite(f)
if drop {
if err := ix.lateMerge(ctx, w); err != nil || w.late != "" {
@@ -71,8 +78,12 @@ func (ix *Indexer) applyManifests(ctx context.Context, f delivery.Forge, drop bo
named[key] = true
cur, has := held[key]
if has && cur.Repository != "" && !delivery.SameRepository(cur.Repository, f.Repository) {
ix.logf("%s at %s: %q names the module %s, which the indexer holds of %s; left as it is (a move between repositories is the operator's)", f.Repository, f.Merge, dir, name, cur.Repository)
continue
if !moves[key] {
ix.waitToMove(key, f, dir)
ix.logf("%s at %s: %q names the module %s, which the indexer holds of %s; left as it is until %s's trunk drops it, and then moved here (a move between repositories is the merge of the group that moves it, ADR 0420 clause 7)", f.Repository, f.Merge, dir, name, cur.Repository, cur.Repository)
continue
}
ix.logf("%s at %s: the module %s moves here from %s, whose trunk no longer holds it", f.Repository, f.Merge, name, cur.Repository)
}
if other, dup := dirOf[name]; dup {
if !has || cur.Path != pathOf(dir) {
@@ -190,11 +201,16 @@ func (ix *Indexer) namedObjects(ctx context.Context, f delivery.Forge) (map[stri
// dropRemoved deletes, from the modules the bucket holds of the repository, each one the trunk no longer holds (issue 1900):
// held(key, module) says whether the trunk holds it. A module of another repository, or of none known, is never touched.
func (ix *Indexer) dropRemoved(ctx context.Context, repository string, held map[string]delivery.Module, trunkHolds func(key string, m delivery.Module) bool) error {
var handed []string
for _, key := range slices.Sorted(maps.Keys(held)) {
cur := held[key]
if cur.Repository == "" || !delivery.SameRepository(cur.Repository, repository) || trunkHolds(key, cur) {
continue
}
if ix.movingTo(key, repository) {
handed = append(handed, key) // written in place by the repository it moves to, never deleted first
continue
}
err := write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) {
if !found {
return nil, false, nil
@@ -209,6 +225,11 @@ func (ix *Indexer) dropRemoved(ctx context.Context, repository string, held map[
}
ix.logf("modules.%s deleted: %s holds no module of that name at its trunk (issue 1900)", key, repository)
}
for _, key := range handed {
if err := ix.handOver(ctx, key); err != nil {
return err
}
}
return nil
}
modules/mesh-delivery-indexer/internal/index/modules_test.go
+59 -14
View File
@@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"path"
"slices"
"strings"
"testing"
"time"
@@ -402,20 +403,64 @@ func TestAMergeWritesOnlyTheModulesItsManifestsName(t *testing.T) { // I1
}
}
func TestAModuleHeldOfAnotherRepositoryIsLeftAndSaid(t *testing.T) { // I1: a move between repositories is the operator's (ADR 0420 clause 7)
ix, mods := modulesIndexer(t)
var logs []string
ix.Logf = func(format string, args ...any) { logs = append(logs, format) }
ctx := context.Background()
if err := ix.Apply(ctx, mergedMsg("novox/mesh-catalog", map[string]string{"modules/a": "t"})); err != nil {
t.Fatal(err)
}
before := string(mods.vals["a"])
if err := ix.Apply(ctx, mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "t2"})); err != nil {
t.Fatal(err)
}
if string(mods.vals["a"]) != before || len(logs) == 0 {
t.Errorf("modules.a = %s (was %s), logs %v: want it left, and said", mods.vals["a"], before, logs)
// deletesSeen is a modules store that remembers each key deleted.
type deletesSeen struct {
*memStore
deleted []string
}
func (d *deletesSeen) Delete(ctx context.Context, key string, rev uint64) error {
d.deleted = append(d.deleted, key)
return d.memStore.Delete(ctx, key, rev)
}
// I1: a move between repositories is the merges of the group that moves it (ADR 0420 clause 7; issue 1126, route-proxy
// leaving the controller's repository). A merge naming a module another repository's trunk still holds leaves it, said; the
// holding repository's merge that drops it hands it over in place, never deleting its key; merged the other way round,
// the module is deleted and then written new.
func TestAModuleMovesBetweenRepositoriesOnlyWhenItsTrunkDropsIt(t *testing.T) {
const from, to = "novox/mesh-controller", "novox/mesh-networking"
held := mergedMsg(from, map[string]string{"modules/a": "t", "modules/b": "t"})
named := mergedMsg(to, map[string]string{"modules/a": "t2"})
dropped := mergedMsg(from, map[string]string{"modules/b": "t"})
for _, tc := range []struct {
name string
merges []delivery.Forge
repository string // modules.a's repository at the end, "" for none
deletedKeyA bool
}{
{"named while its trunk holds it: left", []delivery.Forge{held, named}, from, false},
{"named, then dropped by its trunk: handed over in place", []delivery.Forge{held, named, dropped}, to, false},
{"dropped by its trunk, then named: deleted, then written new", []delivery.Forge{held, dropped, named}, to, true},
{"dropped by its trunk with no merge naming it: deleted", []delivery.Forge{held, dropped}, "", true},
} {
t.Run(tc.name, func(t *testing.T) {
mods := &deletesSeen{memStore: newMem()}
ix := &Indexer{Store: newMem(), Modules: mods, Builds: NewBuilds()}
var logs []string
ix.Logf = func(format string, args ...any) { logs = append(logs, format) }
for _, f := range tc.merges {
if err := ix.Apply(context.Background(), f); err != nil {
t.Fatal(err)
}
}
m, ok := moduleOf(t, mods.memStore, "a")
if got := map[bool]string{true: m.Repository}[ok]; got != tc.repository {
t.Errorf("modules.a of %q, want %q (logs %v)", got, tc.repository, logs)
}
if ok && m.Path != "modules/a" {
t.Errorf("modules.a at %q, want modules/a", m.Path)
}
if deleted := slices.Contains(mods.deleted, "a"); deleted != tc.deletedKeyA {
t.Errorf("modules.a deleted %v, want %v", deleted, tc.deletedKeyA)
}
if _, ok := mods.vals["b"]; !ok {
t.Error("modules.b, which its trunk still holds, is gone")
}
if len(logs) == 0 {
t.Error("nothing was said")
}
})
}
}
modules/mesh-delivery-indexer/internal/index/moves.go
+69
View File
@@ -0,0 +1,69 @@
package index
import (
"context"
"fmt"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// A module moves between repositories by the merges of the delivery group that moves it (novox/hq ADR 0420 clause 7, ADR
// 0501 clause 3; issue 1126: route-proxy leaves the controller's repository, which is archived). The merge of the
// repository it moves to names it while the indexer holds it of another: that merge leaves it, says so, and waits here.
// The merge of the repository that held it, whose trunk no longer holds it, then hands it over: the module is written from
// the waiting merge, in place, so its key never goes missing between the two (dropRemoved). Merged the other way round,
// the first merge deletes the key and the second writes it as a new module. Nothing else moves a module: a merge of a
// repository naming a module whose holding trunk still holds it changes nothing.
//
// The wait is the indexer's memory of a merge it heard, not state of its own: an indexer started between the two merges
// has forgotten it, and the module is deleted by the second; the next merge of the repository it moved to writes it.
// waitToMove keeps, by modules key, the part of a merged message naming one module directory the indexer holds of another
// repository. A later merge of the same repository replaces it.
func (ix *Indexer) waitToMove(key string, f delivery.Forge, dir string) {
one := delivery.Forge{V: f.V, Kind: f.Kind, Repository: f.Repository, Branch: f.Branch, Number: f.Number, Merge: f.Merge, Root: f.Root,
Trees: map[string]string{dir: f.Trees[dir]}, Manifests: map[string]string{dir: f.Manifests[dir]}}
if l := f.Locks[dir]; len(l) > 0 {
one.Locks = map[string]map[string]string{dir: l}
}
if p := f.Paths[dir]; len(p) > 0 {
one.Paths = map[string]map[string]string{dir: p}
}
if p := f.Packages[dir]; len(p) > 0 {
one.Packages = map[string][]string{dir: p}
}
ix.mu.Lock()
defer ix.mu.Unlock()
if ix.moving == nil {
ix.moving = map[string]delivery.Forge{}
}
ix.moving[key] = one
}
// movingTo says a merge of another repository than from waits to take the module at key.
func (ix *Indexer) movingTo(key, from string) bool {
ix.mu.Lock()
defer ix.mu.Unlock()
f, ok := ix.moving[key]
return ok && !delivery.SameRepository(f.Repository, from)
}
// handOver writes the module at key from the merge waiting to take it, and forgets the wait.
func (ix *Indexer) handOver(ctx context.Context, key string) error {
ix.mu.Lock()
f, ok := ix.moving[key]
delete(ix.moving, key)
ix.mu.Unlock()
if !ok {
return nil
}
w, err := ix.applyMoving(ctx, f, false, map[string]bool{key: true})
if err != nil {
return fmt.Errorf("modules.%s, moving to %s: %w", key, f.Repository, err)
}
// A module moved in is judged as a merge's are (ADR 0578 §1, ADR 0579 §1).
if _, err := ix.JudgeAll(ctx, w.fresh); err != nil {
ix.logf("judge: the manifests are not judged after moving %s to %s: %v", key, f.Repository, err)
}
return nil
}
modules/mesh-delivery-indexer/judge/rules.go
+9 -73
View File
@@ -4005,7 +4005,9 @@ func toolsSecretProblems(m catalogue.Manifest) []string {
return problems
}
var BusRemovals = []string{catalogue.RemovesControllerDeadLetterConsumer, catalogue.RemovesEngineActs, catalogue.RemovesAssignmentsStream}
// BusRemovals are what a bus account may remove, the closed set of novox/hq ADR 0564 Limits 4 as ADR 0590 clause 6
// changes it: the node-engine's acts and the ended controller's leftovers.
var BusRemovals = []string{catalogue.RemovesEngineActs, catalogue.RemovesControllerLeftovers}
// ReadsHolderProblems is what a holder of node-reads is held to (novox/hq ADR 0310 §3): it reads as an
// account of its own, so the node's file modes bound what it sees. Carried by the machine's runtime it would
@@ -7251,37 +7253,6 @@ func namesOfAccessIDs(accesses map[string]string) []string {
var renderFuncs = template.FuncMap{"quote": quote}
var defaultSeats = append([]catalogue.Seat{
// The control plane states what it did under the seat it holds (novox/hq ADR 0134): a role's
// events belong to the role, so they keep their address while the holder is replaced. No accepts,
// so no work queue is raised for it — only what its holder may say.
// And it serves the mesh's own verbs as the seat's tools (novox/hq ADR 0154): `status`, `push`,
// `assign` and the rest are a role's interface, not a container's, and stay addressable while
// the control plane is replaced.
{Name: catalogue.ControllerSeatName, Scope: catalogue.ScopeMesh, Decision: "novox/hq ADR 0079",
// And what is wrong, as it changes, and the self-check's heartbeat (novox/hq to-be 45 §2, §4).
Emits: []string{"applied", "refused", "built-before",
"condition-raised", "condition-changed", "condition-cleared", "doctor-heartbeat",
// A value given by hand, replaced after its module's first good start (novox/hq ADR 0228).
"secret-replaced",
// Every act a healer takes (novox/hq to-be 45 §7).
"healer-acted",
// A build put back after its gate failed (novox/hq ADR 0236, to-be 45 §8).
"rolled-back",
// A pull request's merge check, judged (novox/hq to-be 45 §9).
"checked",
// A walk kept in a new state, and a tier sent to a machine: kept until the catalogue's main consumes them no
// more; nothing emits them (ADR 0539).
"plan-moved",
"sent",
// A key of a record mirror written or removed (novox/hq ADR 0289, design 54 §3; design 57 F3).
"assignment-changed", "declared-config-changed", "predefined-changed",
// A builder's key not taken in because the artifact store does not hold an artifact it names (novox/hq
// issue 1937): the builder builds it again.
"artifacts-gone",
// A bucket of a state this release deleted, deleted at the controller's first start (novox/hq ADR 0539
// clause 13): the merge window's, the silent sends' and the building plans' hold, each said once.
"bucket-deleted"},
Serves: catalogue.ControllerVerbs},
// The store's first verbs (novox/hq ADR 0159): the smallest set that makes the store askable,
// served by whichever module holds the seat with tools of these names.
{Name: "mesh-store", Scope: catalogue.ScopeMesh, Delivers: "postgres-database", Decision: "novox/hq ADR 0079",
@@ -7399,21 +7370,8 @@ var defaultSeats = append([]catalogue.Seat{
Serves: []catalogue.Verb{
{Name: "config", Optional: true, Description: "Set, clear, list or read the history of config values: " +
"mesh config or node config for a predefined key, or a module's own value on one node, each set " +
"with why, validated, and naming each module on each node whose value moves. A trusted value only " +
"from the controller's terminal.",
"with why, validated, and naming each module on each node whose value moves.",
Input: configInput()},
// The switch of config to its own sources (novox/hq ADR 0565 §4): optional until every mesh-config serves it.
{Name: "compare", Optional: true, Description: "Every difference between config's own sources and the " +
"controller's records, each with its reason: a module's declared config against the controller's mirror, " +
"the indexer's predefined list against the mirror's, and each open pull request's config candidate against " +
"what the controller told. The take-over waits for none.",
Input: schema(map[string]string{}, nil)},
{Name: "take-over", Optional: true, Description: "Ask the operator's Approve to switch config to its own " +
"sources (the indexer); refused while compare shows a difference.",
Input: schema(map[string]string{"why": "why, in a sentence"}, []string{"why"})},
{Name: "hand-back", Optional: true, Description: "Switch config back to the controller's mirrors, until the " +
"controller hands its lease away; refused after it.",
Input: schema(map[string]string{"why": "why, in a sentence"}, []string{"why"})},
}},
// The provisioner's server (novox/hq ADR 0349 clause 3, design 57 P4): its holder keeps the registry of which
// domain answers which kind of placeholder, and the buckets mesh-provisioner_needs, _applied and _collect,
@@ -7957,14 +7915,6 @@ func placementVerbs() []catalogue.Verb {
"stand, and raise its ask again when its verdict passes and the operator's channel does not hold the ask. " +
"Nothing more: the recovery of a proposal whose judgement or ask was lost.",
Input: schema(map[string]string{"id": "the proposal's id"}, []string{"id"})},
{Name: catalogue.PlacementVerbAdopt, Optional: true, Description: "Take placement over from the controller: its " +
"placement-export is proposed whole as the seed, every node's key, which every node must answer as the same. " +
"Its Approve is the take-over; at the warrant the export is read again and a changed one is proposed anew.",
Input: schema(map[string]string{"why": "why, shown to the operator"}, []string{"why"})},
{Name: catalogue.PlacementVerbHandBack, Optional: true, Description: "The operator's alone, recorded as a hand act: " +
"delete every placement key, the history kept, so every node returns to the controller's compositions. " +
"Refused after the lease write.",
Input: schema(map[string]string{"why": "why, recorded with the hand act"}, []string{"why"})},
}
}
@@ -8004,7 +7954,7 @@ func conditionsVerbs() []catalogue.Verb {
{Name: "conditions", Description: "What is wrong with the mesh now: every open condition, urgent first, then oldest, " +
"raised from the observers' findings (the watchman, the self-check, the healers, the assigner) and cleared when " +
"the finding goes, never by hand. Given a key, that one whole with its evidence; with history, every transition " +
"lately, the controller's before the take-over included.",
"lately.",
Input: schema(map[string]string{
"key": "a condition's key: that one whole; with history, only its transitions",
"scope": "only this scope: machine, plan, call, build, merge, provider, seat, bus, core, probe, mesh, delivery or module",
@@ -8021,17 +7971,6 @@ func conditionsVerbs() []catalogue.Verb {
"why": "why, required and recorded",
"cause": "the cause in a word (the condition's kind when absent)",
}, []string{"key", "for", "why"})},
{Name: "compare", Description: "The keeper's open conditions against the controller's: every key open in one and not " +
"the other, or open in both with another severity, each with its reason; the kinds that end with the controller " +
"and the keys held while their module is delivered listed apart and not counted. The take-over waits for it empty.",
Input: schema(map[string]string{}, nil)},
{Name: "take-over", Description: "Ask the operator's Approve to switch who the router, the grants part and " +
"mesh-watcher hear from the controller to the keeper; refused while compare shows a difference. On the warrant " +
"the keeper writes its take-over key.",
Input: schema(map[string]string{"why": "why now, shown to the operator"}, []string{"why"})},
{Name: "hand-back", Description: "Switch the readers back to the controller's conditions, until the controller's " +
"lease is handed away; refused after it.",
Input: schema(map[string]string{"why": "why, kept with the switch"}, []string{"why"})},
{Name: "resync", Description: "Read every observer's findings again and raise and clear what differs from the open " +
"conditions: the recovery of a keeper that missed a change.",
Input: schema(map[string]string{}, nil)},
@@ -8053,8 +7992,7 @@ func watchdogsVerbs() []catalogue.Verb {
Input: schema(map[string]string{"machine": "one machine"}, nil)},
{Name: "dead-letters", Description: "Every message a consumer on the bus gave up on, kept in DEAD_LETTERS: whose " +
"consumer, the subject, how often it was handed over and when it was given up, newest first. With id: that one " +
"whole. With deliver or drop, a hand act with why, taken only once the controller's lease is handed away; " +
"before, refused naming the controller's verb.",
"whole. With deliver or drop, a hand act with why.",
Input: schema(map[string]string{
"id": "a dead letter's id, as the list gives it: that one whole, with its message",
"consumer": "a consumer's name, or <stream>.<consumer>: only what that one gave up on",
@@ -8089,8 +8027,7 @@ func selfCheckVerbs() []catalogue.Verb {
func healersVerbs() []catalogue.Verb {
return []catalogue.Verb{
{Name: "healers", Description: "The healers: each registered response to one kind of condition, its repair, its " +
"budget and what happens when it is spent; every act taken lately with its outcome, the acts it would have " +
"taken while the controller's lease names a controller, and the mesh-wide brake.",
"budget and what happens when it is spent; every act taken lately with its outcome, and the mesh-wide brake.",
Input: schema(map[string]string{"days": "how many days of acts back (default 7)"}, nil)},
{Name: "heal", Description: "Take one healer's act for one open condition now, within its budget and the brake.",
Input: schema(map[string]string{
@@ -8115,13 +8052,12 @@ func wantsVerbs() []catalogue.Verb {
{Name: "upgrade", Description: "Whether each module's new builds wait for a person: its policy (roll, together or record) " +
"and where it comes from (a person, the module, the bus, its irreplaceable data, the default); the bus is never " +
"rolled. With module, that one; with module, policy and why, a person's choice for it, recorded as a hand act; " +
"with policy default, the choice taken back. With compare, every module whose policy differs from the one the " +
"controller writes beside its compositions.",
"with policy default, the choice taken back. With compare, each node's own composition and the assigner's findings.",
Input: schema(map[string]string{
"module": "one module",
"policy": "with module: roll, together, record, or default to take a person's choice back",
"why": "with policy: why, required and kept",
"compare": "\"true\": every module whose policy at the assigner differs from the controller's",
"compare": "\"true\": each node's own composition, by its digest, and the assigner's findings",
}, nil, "compare")},
{Name: "bus", Description: "What a bus upgrade would do: the fingerprint of the bus module each node runs against the " +
"indexer's newest, and the last bus steps from the hand-act log and the assigner's puts of the bus. Read-only: " +
modules/mesh-delivery-indexer/judge/rules_test.go
+17 -15
View File
@@ -2588,7 +2588,7 @@ func TestAProcessDeclaresItsBusAndIsAccepted(t *testing.T) {
for _, c := range []struct {
name, manifest, refused string
}{
{name: "the bus module's tools removing", manifest: tools(broker, `{"removes":["DEAD_LETTER_NOTICES.controller","mesh-host_acts","ASSIGNMENTS"]}`)},
{name: "the bus module's tools removing", manifest: tools(broker, `{"removes":["mesh-host_acts","controller-leftovers"]}`)},
{name: "a removal outside the set", manifest: tools(broker, `{"removes":["EVENTS.a_m"]}`), refused: "not one of"},
{name: "a removal beside a grant", manifest: tools(broker, `{"removes":["mesh-host_acts"],"invokes":["nats.nats_streams"]}`), refused: "with nothing else"},
{name: "another module's tools removing", manifest: tools(`[]`, `{"removes":["mesh-host_acts"]}`), refused: "only the bus module"},
@@ -2613,7 +2613,7 @@ func TestAProcessDeclaresItsBusAndIsAccepted(t *testing.T) {
}
accounts := m.BusAccounts()
if len(accounts) != 1 || accounts[0].ID != "nats-tools" || !accounts[0].Tools ||
!strSliceEqual(accounts[0].Bus.Removes, []string{catalogue.RemovesControllerDeadLetterConsumer, catalogue.RemovesEngineActs, catalogue.RemovesAssignmentsStream}) {
!strSliceEqual(accounts[0].Bus.Removes, []string{catalogue.RemovesEngineActs, catalogue.RemovesControllerLeftovers}) {
t.Errorf("%s: the accounts are %+v", c.name, accounts)
}
// Resolved, the build goes and its bundles keep the bus (build.go).
@@ -4357,7 +4357,7 @@ func TestTheBussAccountSaysNothingOnTheBus(t *testing.T) {
"invokes":["seat:mesh-hand-acts.hand-act"],"consumes":[],
"tools":["nats_server","nats_bucket_expiry"],
"build":{"artifacts":[{"name":"tools","kind":"bundle","language":"go","system":"arch","from":"cmd/nats-tools",
"binary":"nats-tools","loads":["nats-tools"],"bus":{"removes":["DEAD_LETTER_NOTICES.controller","ASSIGNMENTS"]}}]}}`)
"binary":"nats-tools","loads":["nats-tools"],"bus":{"removes":["controller-leftovers"]}}]}}`)
nm, err := ParseManifest(nats)
if err != nil {
t.Fatalf("the bus module whose tools bundle invokes a seat verb was refused: %v", err)
@@ -4970,9 +4970,10 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) {
delivered[s.Delivers] = s.Name
}
}
// Forty-seven with mesh-placement (novox/hq ADR 0567), serving its seven verbs.
// Forty-seven with mesh-placement (novox/hq ADR 0567), serving its five verbs: adopt and hand-back went with the
// controller (novox/hq ADR 0590, issue 1126).
if s, ok := catalogue.SeatNamed(catalogue.PlacementSeat); !ok || s.Scope != catalogue.ScopeMesh || !reflect.DeepEqual(catalogue.VerbNames(s.Serves),
[]string{"placement", "proposals", "propose", "withdraw", "again", "adopt", "hand-back"}) {
[]string{"placement", "proposals", "propose", "withdraw", "again"}) {
t.Errorf("the placement seat is %v, %v", ok, s)
}
// The checker's seat serves the try, which replaces the shell's merge check (ADR 0547 §10, ADR 0245).
@@ -4993,12 +4994,12 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) {
// The four health seats (novox/hq ADR 0564 clause 1): at mesh scope, each held once, each naming ADR 0564, with the
// verbs and events the record names; the events keep the controller's names (clause 3).
for _, c := range []struct{ seat, verbs, emits string }{
{catalogue.ConditionsSeat, "conditions,silence,compare,take-over,hand-back,resync", "condition-raised,condition-changed,condition-cleared"},
{catalogue.ConditionsSeat, "conditions,silence,resync", "condition-raised,condition-changed,condition-cleared"},
{catalogue.WatchdogsSeat, "signals,stuck,presence,dead-letters,resync", ""},
{catalogue.SelfCheckSeat, "doctor,probes", "checked"},
{catalogue.HealersSeat, "healers,heal,drill", "healer-acted"},
// And mesh-config's switch (ADR 0565 §4), beside its config.
{"mesh-config", "config,compare,take-over,hand-back", ""},
// And mesh-config's config; its switch went with the controller (ADR 0590 clause 3).
{"mesh-config", "config", ""},
// And ADR 0565 clause 1's two.
{catalogue.WantsSeat, "upgrade,bus", "policy-changed"},
{catalogue.HandActsSeat, "hand-act,hand-acts,warranted", "act-recorded"},
@@ -5049,9 +5050,10 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) {
// Forty-eight with mesh-delivery-checker (novox/hq ADR 0547 §10, issue 774); forty-nine with node-engine (novox/hq ADR 0563 §1);
// fifty-three with the four health seats (novox/hq ADR 0564 clause 1); fifty-five with mesh-wants and mesh-hand-acts
// (novox/hq ADR 0565 clause 1); fifty-six with mesh-artifact-sweep (novox/hq ADR 0571 clause 2);
// fifty-seven with mesh-grants (ADR 0582 clause 1); fifty-eight with mesh-enrolment (novox/hq ADR 0587 §1).
if len(catalogue.Seats()) != 58 {
t.Errorf("the mesh defines %d seats rather than 58; the set is closed, so a change here is "+
// fifty-seven with mesh-grants (ADR 0582 clause 1); fifty-eight with mesh-enrolment (novox/hq ADR 0587 §1); fifty-six
// without the controller's seat and mesh-grants, which served its lease write alone (novox/hq ADR 0590, issue 1126).
if len(catalogue.Seats()) != 56 {
t.Errorf("the mesh defines %d seats rather than 56; the set is closed, so a change here is "+
"a decision (novox/hq ADR 0110): %s", len(catalogue.Seats()), seatNames())
}
}
@@ -5286,10 +5288,10 @@ func TestUsingASeatNobodyDeclaresIsRefused(t *testing.T) {
catalogue.Manifest{Module: "shop", Invokes: []string{"seat:telegram-sender.status"}}, unclaimed, true},
{"a verb of a mesh seat whose holder lives in another repository", "",
catalogue.Manifest{Module: "shop", Invokes: []string{"seat:mesh-watchdogs.signals"}}, telegram(), false},
{"a verb a mesh seat does not promise", `the seat mesh-controller serves no verb "root-freed"`,
catalogue.Manifest{Module: "shop", Invokes: []string{"seat:mesh-controller.root-freed"}}, telegram(), false},
{"the controller's verb: the controller answers its seat", "",
catalogue.Manifest{Module: "shop", Invokes: []string{"seat:mesh-controller.root-free"}}, telegram(), false},
{"a verb a mesh seat does not promise", `the seat mesh-placement serves no verb "adopt"`,
catalogue.Manifest{Module: "shop", Invokes: []string{"seat:mesh-placement.adopt"}}, telegram(), false},
{"the controller's seat is gone (ADR 0590)", `a verb of the seat "mesh-controller", ` + catalogue.UndeclaredSeat,
catalogue.Manifest{Module: "shop", Invokes: []string{"seat:mesh-controller.root-free"}}, telegram(), true},
{"the engine's verb: every node's engine holds its seat", "",
catalogue.Manifest{Module: "shop", Invokes: []string{"seat:node-engine.root-free"}}, telegram(), false},
{"a verb the engine's seat does not promise", `the seat node-engine serves no verb "root-freed"`,
modules/mesh-delivery-teller/go.mod
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-teller
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
modules/mesh-delivery-teller/go.sum
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 h1:x8k7IeMirMTxUo8BH+xUO+MDnHYgU+636jl1NIw1wvs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791 h1:7SYIDPMxbPBr8dMB6k9fYsgVt4ZyWVL41q4LMFsfAvs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go
+1 -2
View File
@@ -219,8 +219,7 @@ func Tools(r view.Reader, u view.UsageReader) []stdio.Tool {
{Name: view.TrustsReadTool,
Description: "What every node trusts a mesh signature from, in one call (novox/hq ADR 0550 §2): each node's record's trusts " +
"(from node-engine_have <node>.node), whether it names the vault's signing key as mesh-vault publishes it in its " +
"state signers and whether it still names the controller's, the nodes that do not name the vault's yet (not_yet), " +
"and every: true once all do, the gate before the controller's lease is handed away. Read-only.",
"state signers, the nodes that do not name the vault's yet (not_yet), and every: true once all do. Read-only.",
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) {
return r.Trusts()
modules/mesh-delivery-view/go.mod
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-view
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
modules/mesh-delivery-view/go.sum
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 h1:x8k7IeMirMTxUo8BH+xUO+MDnHYgU+636jl1NIw1wvs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791 h1:7SYIDPMxbPBr8dMB6k9fYsgVt4ZyWVL41q4LMFsfAvs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
modules/mesh-delivery-view/internal/view/read.go
+4 -8
View File
@@ -757,15 +757,14 @@ func (r Reader) Haves(node, module string) (KeptAnswer, error) {
}
// TrustsReadTool reads what every node trusts a mesh signature from, in one call (novox/hq ADR 0550 §2; design 47's L3
// s6b): the gate before the lease write is that every node's record names the vault's key.
// s6b): whether every node's record names the vault's key.
const TrustsReadTool = "node_trusts"
// NodeTrusts is what one node's record says it trusts: its lines (delivery.Trusts) and whether they name the vault's key
// the vault publishes and the controller's key.
// the vault publishes.
type NodeTrusts struct {
Trusts []string `json:"trusts"`
TrustsVault bool `json:"trusts_vault"`
TrustsController bool `json:"trusts_controller"`
Trusts []string `json:"trusts"`
TrustsVault bool `json:"trusts_vault"`
}
// TrustsAnswer is node_trusts' answer: the vault's key as it publishes it, each node's trusts, the nodes whose record does
@@ -824,9 +823,6 @@ func (r Reader) Trusts() (TrustsAnswer, error) {
if h.Record != nil {
t.Trusts = h.Record.Trusts
}
for _, line := range t.Trusts {
t.TrustsController = t.TrustsController || strings.HasPrefix(line, delivery.SignerController+" key ")
}
t.TrustsVault = out.VaultKey != "" && delivery.TrustsKey(t.Trusts, delivery.SignerVault, out.VaultKey)
out.Nodes[node] = t
if !t.TrustsVault {
modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go
+7
View File
@@ -37,6 +37,13 @@ func TestRunNeedsItsCredential(t *testing.T) { // 1135/AC6
if routeOf("mesh.mod.web.event.started") != "" {
t.Error("another module's event goes to a row")
}
// 1126/B9 (ADR 0590 §3): the lease write's events went with it; the grants part says no lease event any more.
for _, gone := range []string{"mesh.seat.mesh-grants.event.lease-altered", "mesh.seat.mesh-grants.event.removal-failed",
"$JS.EVENT.ADVISORY.STREAM.DELETED.ASSIGNMENTS"} {
if r := routeOf(gone); r != "" && r != routeAdvisory {
t.Errorf("%s goes to the row %s", gone, r)
}
}
})
err := run(context.Background(), func(string) string { return "" }, io.Discard)
if err == nil || !strings.Contains(err.Error(), "MESH_BROKER_FILE") {
modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/signals.go
-27
View File
@@ -13,7 +13,6 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/health"
"mesh-delivery-watchman/internal/deadletters"
@@ -39,13 +38,6 @@ const (
routeDead = "dead-letter"
routeBusFault = "bus-fault"
routeProv = "provisioned"
routeLease = "lease"
)
// The lease write's subjects (ADR 0582 clauses 4, 6): the seat mesh-grants' events, and ASSIGNMENTS deleted.
var (
leaseEventPrefix = "mesh.seat." + catalogue.GrantsSeat + ".event."
assignmentsGone = "$JS.EVENT.ADVISORY.STREAM.DELETED." + buscompose.AssignmentsStream
)
// The bus module's events of the connections the bus closed (S9).
@@ -69,8 +61,6 @@ func routeOf(subject string) string {
return routeTools
case subject == CheckedSubject:
return routeChecked
case strings.HasPrefix(subject, leaseEventPrefix) || subject == assignmentsGone:
return routeLease
case strings.HasPrefix(subject, "$JS.EVENT.ADVISORY."):
return routeAdvisory
case strings.HasPrefix(subject, "mesh.events.dead."):
@@ -106,7 +96,6 @@ func heard(w *watch.Watchman, dead *deadKeeper, m *nats.Msg, logf func(string, .
}
if dead.isTheControllersNotices(a.ID) {
dead.check() // the controller's consumer removed: the watchman may take the notices now
w.RemovalGone(watch.RemovalConsumer)
return
}
w.Advised(a)
@@ -121,22 +110,6 @@ func heard(w *watch.Watchman, dead *deadKeeper, m *nats.Msg, logf func(string, .
kind = watch.KindConnectionClosed
}
w.Closed(kind, c)
case routeLease:
var body struct {
Altered string `json:"altered"`
Failed string `json:"failed"`
}
_ = json.Unmarshal(m.Data, &body)
switch m.Subject {
case assignmentsGone:
w.RemovalGone(watch.RemovalStream)
case leaseEventPrefix + catalogue.GrantsEventLeaseAltered:
w.LeaseAltered(body.Altered)
case leaseEventPrefix + catalogue.GrantsEventLeaseHanded:
w.LeaseHanded()
case leaseEventPrefix + catalogue.GrantsEventRemovalFailed:
w.RemovalFailed(body.Failed)
}
case routeProv:
var r watch.ProvReport
if err := json.Unmarshal(m.Data, &r); err != nil {
modules/mesh-delivery-watchman/go.mod
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-watchman
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
modules/mesh-delivery-watchman/go.sum
+2 -6
View File
@@ -1,9 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 h1:x8k7IeMirMTxUo8BH+xUO+MDnHYgU+636jl1NIw1wvs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c h1:/YCxh+NYfqry3PxZYY1nQXaccupobxpsFt8c9BYuGBg=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21 h1:4I4ayo0fZYP4PBfEX1sGDDd6arnw5IQJySJ3jcFobPM=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791 h1:7SYIDPMxbPBr8dMB6k9fYsgVt4ZyWVL41q4LMFsfAvs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E=
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
modules/mesh-delivery-watchman/internal/deadletters/deadletters.go
+7 -10
View File
@@ -1,13 +1,11 @@
// Package deadletters is what a consumer gave up on, kept by the watchman from the lease write (novox/hq ADR 0564 §5,
// Package deadletters is what a consumer gave up on, kept by the watchman since the controller's end (novox/hq ADR 0564 §5,
// ADR 0264, ADR 0376, issue 1952 WM5): ported from mesh-controller internal/link/deadletters.go at e04846e9 with nothing
// changed in what is kept, aged or withheld, on the jetstream API.
//
// DEAD_LETTER_NOTICES is a work queue that holds one consumer. The controller's consumer `controller` never expires, so
// the watchman makes its own (Consumer) only when the lease names no controller and `controller` is gone (Gate); while
// `controller` still stands after the lease write it says so as a finding. Removing `controller` is a step of the lease
// write, with the bus module's tool. Before the lease write the watchman only lists and shows DEAD_LETTERS, which the
// controller still fills, and refuses to deliver or drop, naming the controller's verb; from it, delivering and dropping
// wait for the hand-act log's home (issue 1953), so they are refused naming that.
// DEAD_LETTER_NOTICES is a work queue that holds one consumer. The ended controller's consumer `controller` never
// expires, so the watchman makes its own (Consumer) only once `controller` is gone (Gate); while it stands the watchman says
// so as a finding. Removing `controller` is a hand act with the bus module's tool (novox/hq ADR 0590 clause 6). Delivering
// and dropping wait for the hand-act log's home (issue 1953), so they are refused naming that.
package deadletters
import (
@@ -317,9 +315,8 @@ func Refusal(act string) error {
}
// Gate is whether the watchman takes the notices itself: once the controller's consumer is gone. waiting says `controller`
// still stands, which is the watchman's finding: each notice waits in the queue until that consumer is removed. The lease
// is not read: the controller ended by its unassignment, never by the lease write (issue 1126), and mesh-sdk's
// buscompose.LeaseHandedAway answers false for the absent key, which would keep the notices the controller's for good.
// still stands, which is the watchman's finding: each notice waits in the queue until that consumer is removed (the
// controller ended by its unassignment, issue 1126).
func Gate(controllerStands bool) (take, waiting bool) {
return !controllerStands, controllerStands
}
modules/mesh-delivery-watchman/internal/watch/advisories.go
+4 -2
View File
@@ -8,7 +8,7 @@ import (
// The bus's advisories the watchman hears (S9, mesh-controller internal/link/advisories.go at e04846e9): a durable consumer
// the mesh defines deleted is `consumer-lost`. A message given up on is said from DEAD_LETTERS, where it is kept, for as
// long as it is kept (DeadLettersHeld), not from its advisory. The controller's own slow consumer and refused subjects end
// long as it is kept (DeadLettersHeld), not from its advisory. The controller's own slow consumer and refused subjects ended
// with it (ADR 0564 §4).
// Advisory subjects the watchman subscribes.
@@ -48,7 +48,9 @@ func MeshNamed(stream, name string) bool {
case strings.HasPrefix(stream, "SEAT_"):
return strings.HasSuffix(name, "_worker")
case name == "controller" || name == "controller-merges":
return true
// The ended controller's consumers are removed as hand acts (novox/hq ADR 0590 clause 6), never lost; its consumer on
// DEAD_LETTER_NOTICES alone is read, whose removal tells the watchman to take the notices.
return stream == "DEAD_LETTER_NOTICES" && name == "controller"
case stream == "EVENTS":
return strings.Contains(name, "_")
}
modules/mesh-delivery-watchman/internal/watch/dead.go
+9 -9
View File
@@ -8,15 +8,15 @@ import (
)
// The dead letters' findings (novox/hq ADR 0564 §5, issue 330): a consumer DEAD_LETTERS holds a message for is said for as
// long as it holds any, `bus.<stream>.<consumer>.max-deliveries`, and the notices waiting behind the controller's consumer
// after the lease write are said as `bus.DEAD_LETTER_NOTICES.controller.notices-waiting`.
// long as it holds any, `bus.<stream>.<consumer>.max-deliveries`, and the notices waiting behind the ended controller's
// consumer are said as `bus.DEAD_LETTER_NOTICES.controller.notices-waiting`.
// DeadRecountEvery is how often DEAD_LETTERS is counted again while a consumer's finding stands: a letter delivered again
// or dropped by the controller's verb, which the watchman does not hear, ends its finding within it. The expiry of the
// watchman's own pulse, never a loop.
const DeadRecountEvery = 5 * time.Minute
// KindNoticesWaiting is the finding of notices waiting while the controller's consumer stands after the lease write.
// KindNoticesWaiting is the finding of notices waiting while the ended controller's consumer stands.
const KindNoticesWaiting = "notices-waiting"
// DeadLettersHeld is DEAD_LETTERS' count per consumer, `<stream>.<consumer>`, read whole: each consumer held writes its
@@ -73,16 +73,16 @@ func consumerMachine(stream, consumer string) string {
return ""
}
// NoticesWaiting says whether the lease is handed away while the controller's consumer on DEAD_LETTER_NOTICES still
// stands: every notice then waits in the queue, kept a week, until the lease write's step removes the consumer.
// NoticesWaiting says whether the ended controller's consumer on DEAD_LETTER_NOTICES still stands: every notice then waits
// in the queue, kept a week, until that consumer is removed (novox/hq ADR 0590 clause 6).
func (w *Watchman) NoticesWaiting(waiting bool) {
w.mu.Lock()
defer w.mu.Unlock()
f := Finding{Scope: ScopeBus, ID: "DEAD_LETTER_NOTICES.controller", Kind: KindNoticesWaiting, Severity: Warning, Source: "dead-letters",
Summary: "the lease names no controller but the controller's consumer on DEAD_LETTER_NOTICES still stands, so every " +
"notice of a message given up on waits in the queue, kept a week, and nothing keeps it in DEAD_LETTERS",
Said: "the consumer controller on DEAD_LETTER_NOTICES stands after the lease write",
Needs: "remove the consumer controller on DEAD_LETTER_NOTICES with the bus module's tool, the lease write's step."}
Summary: "the controller is unassigned but its consumer on DEAD_LETTER_NOTICES still stands, so every notice of a " +
"message given up on waits in the queue, kept a week, and nothing keeps it in DEAD_LETTERS",
Said: "the consumer controller on DEAD_LETTER_NOTICES stands after the controller's end",
Needs: "remove the consumer controller on DEAD_LETTER_NOTICES with the bus module's tool, recorded as a hand act (ADR 0590)."}
if waiting {
w.find(f)
return
modules/mesh-delivery-watchman/internal/watch/findings_test.go
+24 -20
View File
@@ -59,23 +59,6 @@ func TestTheWatchmansFindings(t *testing.T) {
}
w.ModuleVerdict("node-tools", value(""), Put)
}, map[string]string{}},
{"774/S22/AC1, AC2: lease-altered and removal-failed are findings; lease-handed and the removal read gone clear them", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
w.LeaseAltered("the key holder names controller@a pid 9, not mesh-grants by lease-write-1")
if f := o.findings[FindingKey(ScopeBus, "mesh-controller_lease", KindLeaseAltered)]; !strings.Contains(f.Summary, "controller@a pid 9") {
t.Fatalf("findings = %v", o.findings)
}
w.LeaseHanded()
w.RemovalFailed("the stream ASSIGNMENTS: ASSIGNMENTS has 1 consumers")
k := FindingKey(ScopeBus, "lease-removal", KindRemovalFailed)
if f := o.findings[k]; !strings.Contains(f.Summary, "ASSIGNMENTS has 1 consumers") {
t.Fatalf("findings = %v", o.findings)
}
w.RemovalGone(RemovalConsumer) // not what it waits on
if _, ok := o.findings[k]; !ok {
t.Fatal("cleared by the consumer, while the stream it names stands")
}
w.RemovalGone(RemovalStream)
}, map[string]string{}},
{"WM4: a stuck key not moved is its finding, and its move deletes it", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
w.Step("go", "n1.ask1", runningStep("ask1", t0), Put)
expire(w, c, pulseKeyOf(t, checksKey, "ask1"), 120*time.Second)
@@ -157,6 +140,18 @@ func TestTheWatchmansFindings(t *testing.T) {
if _, ok := ReadAdvisory("$JS.EVENT.ADVISORY.CONSUMER.DELETED.KV_x.abc", []byte(`{"stream":"KV_x","consumer":"abc"}`), nil); ok {
t.Fatal("a reader's own consumer is said")
}
// 1126/B9 (ADR 0590 clause 6): the ended controller's consumers are removed as hand acts, never lost; only its
// consumer on DEAD_LETTER_NOTICES is still read, which tells the watchman to take the notices.
for _, gone := range [][2]string{{"EVENTS", "controller"}, {"EVENTS", "controller-merges"}, {"CONTROL", "controller"}} {
body := []byte(`{"stream":"` + gone[0] + `","consumer":"` + gone[1] + `"}`)
if _, ok := ReadAdvisory("$JS.EVENT.ADVISORY.CONSUMER.DELETED."+gone[0]+"."+gone[1], body, func(s, c string) string { return c }); ok {
t.Errorf("the removal of %s.%s is said as a consumer lost", gone[0], gone[1])
}
}
if _, ok := ReadAdvisory("$JS.EVENT.ADVISORY.CONSUMER.DELETED.DEAD_LETTER_NOTICES.controller",
[]byte(`{"stream":"DEAD_LETTER_NOTICES","consumer":"controller"}`), func(s, c string) string { return c }); !ok {
t.Error("the removal of the controller's consumer on DEAD_LETTER_NOTICES is not read")
}
w.Advised(a)
w.Advised(a)
k := "bus.EVENTS.ace_web.consumer-lost"
@@ -254,7 +249,7 @@ func TestTheWatchmansFindings(t *testing.T) {
}
return b
}
w.Have("novox.mesh-controller", haveOf(t, delivery.PhaseRunning, delivery.HealthOK, ""), Put) // the control node
w.Have("novox.nats", haveOf(t, delivery.PhaseRunning, delivery.HealthOK, ""), Put) // the control node: the bus node
w.NodeHealth("novox", report(delivery.NodeHealth{Node: "novox", Network: &delivery.NetworkHealth{State: "unhealthy", Parts: []delivery.NetworkPart{
{Part: "resolv-conf", State: "unhealthy", Writer: "NetworkManager", Owner: "systemd-resolved", Said: "rewritten"},
{Part: "names", State: "unhealthy", Reason: "names do not resolve.", Said: "x"},
@@ -346,7 +341,16 @@ func TestTheWatchmansFindings(t *testing.T) {
w.Have("ace.mail", haveOf(t, delivery.PhaseRemoved, delivery.HealthUnknown, ""), Put)
}, map[string]string{}},
{"WM2 S1: a control node silent is urgent after half an hour", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
w.Have("novox.mesh-controller", haveOf(t, delivery.PhaseRunning, delivery.HealthOK, ""), Put)
// 1126 (ADR 0590): the control node is the bus node, the one whose have holds the bus module; a have of the
// ended controller names none.
w.Have("g14.mesh-controller", haveOf(t, delivery.PhaseRunning, delivery.HealthOK, ""), Put)
w.Connect("node.g14")
w.Disconnect("node.g14")
expire(w, c, pulseKeyOf(t, presenceNS, "g14"), 120*time.Second)
if _, ok := w.signals[signalKey("S1", "g14")]; ok {
t.Fatalf("a node running the controller's have is held a control node")
}
w.Have("novox.nats", haveOf(t, delivery.PhaseRunning, delivery.HealthOK, ""), Put)
w.Connect("node.novox")
w.Disconnect("node.novox")
expire(w, c, pulseKeyOf(t, presenceNS, "novox"), 120*time.Second)
@@ -354,7 +358,7 @@ func TestTheWatchmansFindings(t *testing.T) {
t.Fatalf("findings = %v", o.findings)
}
expire(w, c, signalKey("S1", "novox"), ControlNodeUrgentAfter)
}, map[string]string{"machine.novox.silent": Urgent}},
}, map[string]string{"machine.novox.silent": Urgent, "machine.g14.silent": Warning}},
{"WM3 S9: the bus module's closures, each counted once, ended an hour after the newest", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
cl := BusClosure{User: "ace.web", CID: 7, Reason: "Slow Consumer (Write Deadline)", Opened: t0.Add(-time.Hour), Closed: t0}
w.Closed(KindSlowConsumer, cl)
modules/mesh-delivery-watchman/internal/watch/lease.go
-80
View File
@@ -1,80 +0,0 @@
package watch
import (
"strings"
)
// **The lease write's findings** (novox/hq ADR 0582 clauses 4 and 6, issue 774): mesh-grants says lease-altered when the
// lease key or its bucket's expiry differs from what it wrote, and removal-failed when the day-later removal of the
// controller's consumer of DEAD_LETTER_NOTICES or the stream ASSIGNMENTS failed; the watchman raises each as a finding.
// lease-altered clears when lease-handed is said again (the operator asked the write again); removal-failed clears when
// what it names reads gone: the consumer's deletion advisory and ASSIGNMENTS' deletion advisory, each heard.
// The two kinds.
const (
KindLeaseAltered = "lease-altered"
KindRemovalFailed = "removal-failed"
)
// The finding's IDs: the lease's bucket, and the removal.
const (
leaseID = "mesh-controller_lease"
removalID = "lease-removal"
)
// What a failed removal still waits on.
const (
RemovalConsumer = "DEAD_LETTER_NOTICES.controller"
RemovalStream = "ASSIGNMENTS"
)
// LeaseAltered raises the finding that the lease differs from what mesh-grants wrote, naming what changed.
func (w *Watchman) LeaseAltered(what string) {
w.mu.Lock()
defer w.mu.Unlock()
w.find(Finding{Scope: ScopeBus, ID: leaseID, Kind: KindLeaseAltered, Severity: Warning, Source: "mesh-grants",
Summary: "the controller's lease differs from what mesh-grants wrote: " + what,
Said: what,
Needs: "a look at who changed the lease; mesh-grants writes nothing until lease-write is asked again.",
Resolved: "lease-write is asked and warranted again, and the lease reads as written"})
}
// LeaseHanded clears the lease-altered finding: the lease reads as written again.
func (w *Watchman) LeaseHanded() {
w.mu.Lock()
defer w.mu.Unlock()
w.unfind(FindingKey(ScopeBus, leaseID, KindLeaseAltered))
}
// RemovalFailed raises the finding that the day-later removal failed, naming what failed; it waits on each of the
// consumer and the stream the failure names.
func (w *Watchman) RemovalFailed(failed string) {
w.mu.Lock()
defer w.mu.Unlock()
if w.removalWaits == nil {
w.removalWaits = map[string]bool{}
}
for _, what := range []string{RemovalConsumer, RemovalStream} {
if strings.Contains(failed, strings.SplitN(what, ".", 2)[0]) {
w.removalWaits[what] = true
}
}
if len(w.removalWaits) == 0 { // names neither: waits on both
w.removalWaits[RemovalConsumer], w.removalWaits[RemovalStream] = true, true
}
w.find(Finding{Scope: ScopeBus, ID: removalID, Kind: KindRemovalFailed, Severity: Warning, Source: "mesh-grants",
Summary: "the day-later removal of the controller's consumer and stream failed: " + failed,
Said: failed,
Needs: "a look at why; mesh-grants tries again in a day, and grants_removal_due starts it sooner.",
Resolved: "the controller's consumer of DEAD_LETTER_NOTICES and the stream ASSIGNMENTS read gone"})
}
// RemovalGone hears one of the two removed (RemovalConsumer or RemovalStream), and clears the finding once neither waits.
func (w *Watchman) RemovalGone(what string) {
w.mu.Lock()
defer w.mu.Unlock()
delete(w.removalWaits, what)
if len(w.removalWaits) == 0 {
w.unfind(FindingKey(ScopeBus, removalID, KindRemovalFailed))
}
}
modules/mesh-delivery-watchman/internal/watch/signals.go
+7 -5
View File
@@ -252,7 +252,7 @@ func (w *Watchman) stuckFinding(sk string, s delivery.Stuck) {
f := Finding{Scope: ScopeMachine, ID: node, Kind: KindSilent, Machine: node, Severity: Warning, Source: "S1",
Summary: fmt.Sprintf("%s has not been heard from since %s (bound %s)", node, stamp(s.Since), SilentBound),
Said: fmt.Sprintf("offline since %s", s.Since.UTC().Format(time.RFC3339)), Since: s.Since}
// A control node, the machine running the controller, is urgent after half an hour silent, as the controller
// A control node, the bus node (ControlNodeModule), is urgent after half an hour silent, as the controller
// says it (its watchHeartbeats): the expiry of a pulse from its silence's start, never a loop.
if w.control[node] {
if age := w.now().Sub(s.Since); age > ControlNodeUrgentAfter {
@@ -459,14 +459,14 @@ func ToolsNode(subject string) string {
// ---- S21 and unhealthy, from the haves and the provisioner clients' reports ----
// haveFindings reads a have's change for the rows the haves feed: S21 (the machine's applies failing, applyCheck),
// unhealthy (two writes in a row, urgent after four hours), and where the controller runs (S1's control node). A have
// unhealthy (two writes in a row, urgent after four hours), and where the bus module runs (S1's control node). A have
// removed or deleted ends its part of them.
//
// needs-operator is not observed from the haves: a module's part that waits for the operator is a provision in the state
// `waits` of the seat mesh-provisioner (Provision, ADR 0570 §7).
func (w *Watchman) haveFindings(key string, node, module string, h *delivery.Have) {
unhealthyKey := FindingKey(ScopeModule, module+"."+node, "unhealthy")
if module == ControllerModule {
if module == ControlNodeModule {
w.control[node] = h != nil && h.Phase != delivery.PhaseRemoved
}
if h == nil || h.Phase == delivery.PhaseRemoved {
@@ -531,8 +531,10 @@ func (w *Watchman) haveFindings(key string, node, module string, h *delivery.Hav
}
}
// ControllerModule is the module whose have names a control node.
const ControllerModule = "mesh-controller"
// ControlNodeModule is the module whose have names a control node: the bus module, so the control node is the bus node,
// the one stably reachable node every part depends on (novox/hq ADR 0590, issue 1126: the controller that named it ended).
// The same name the bus module's events are heard under (SlowConsumerSubject).
const ControlNodeModule = "nats"
// Provision is a change of a key of mesh-provisioner's `provisions` (ADR 0570 §7): a provision in `waits` is a part of its
// consumer waiting for the operator, said as `module.<consumer>.<node>.needs-operator` naming each waiting provision and
modules/mesh-delivery-watchman/internal/watch/watch.go
-2
View File
@@ -86,8 +86,6 @@ type Watchman struct {
recount func()
// waiting is, by `<consumer>.<node>`, each of its provisions in waits, with its words.
waiting map[string]map[string]string
// removalWaits is what a failed removal of the lease write still waits on to read gone (lease.go).
removalWaits map[string]bool
// health is each node's own health report, and tunnels each node's tunnel key from its record.
health map[string]delivery.NodeHealth
tunnels map[string]string
modules/mesh-enrolment/cmd/mesh-enrolment/holder.go
-7
View File
@@ -158,13 +158,6 @@ func checkTunnelKey(k string) error {
return nil
}
// issuing says whether issue is on, and why: always, since the controller ended by its unassignment, never by the lease
// write (issue 1126), so the seat is the only issuer (ADR 0587 §4). The lease is no longer read: mesh-sdk's
// buscompose.LeaseHandedAway answers false for the absent key, which would keep issuing off for good.
func (h *Holder) issuing() (bool, string) {
return true, "the seat is the only issuer: the controller ended by its unassignment (issue 1126)"
}
// openEnrolments reads every node key of the bucket.
func (h *Holder) openEnrolments() (map[string]Row, map[string]Enrolment, error) {
keys, err := h.Store.Keys()
modules/mesh-enrolment/cmd/mesh-enrolment/tokens.go
+1 -10
View File
@@ -26,12 +26,6 @@ type TokenRow struct {
Unsealed string `json:"unsealed,omitempty"`
}
// Issuing is whether issue is on, and why.
type Issuing struct {
On bool `json:"on"`
Why string `json:"why"`
}
// Fill is how full the bucket is: its keys and the bytes of their values (the state's cap is the bus's, which no bundle
// can read, so the fill is stated without it).
type Fill struct {
@@ -42,14 +36,13 @@ type Fill struct {
// TokensAnswer is what `tokens` answers.
type TokensAnswer struct {
Open []TokenRow `json:"open"`
Issuing Issuing `json:"issuing"`
Fill Fill `json:"fill"`
History []json.RawMessage `json:"history,omitempty"`
// HistoryFrom says where the history comes from, when one was asked for.
HistoryFrom string `json:"history-from,omitempty"`
}
// Tokens answers one open enrolment or all, with the bucket's fill, whether issuing is on and why, and with history the
// Tokens answers one open enrolment or all, with the bucket's fill, and with history the
// ended enrolments the holder's own events say (ADR 0587 §8).
func (h *Holder) Tokens(node string, history bool) (TokensAnswer, error) {
rows, open, err := h.openEnrolments()
@@ -57,8 +50,6 @@ func (h *Holder) Tokens(node string, history bool) (TokensAnswer, error) {
return TokensAnswer{}, err
}
var out TokensAnswer
on, why := h.issuing()
out.Issuing = Issuing{On: on, Why: why}
if keys, err := h.Store.Keys(); err == nil {
out.Fill.Keys = len(keys)
for _, k := range keys {
modules/mesh-enrolment/cmd/mesh-enrolment/tokens_test.go
+8 -7
View File
@@ -5,6 +5,7 @@ import (
"os"
"path/filepath"
"slices"
"strings"
"testing"
"time"
@@ -25,7 +26,7 @@ func (r *rig) seed(node, address string, stage string) {
}
}
// 774/S54/AC1 and AC2: tokens answers each stage with its expiry, ask and revision, the fill and whether issuing is on;
// 774/S54/AC1 and AC2: tokens answers each stage with its expiry, ask and revision, and the fill;
// for one node at live it relays the vault's sealed token and keeps nothing; and the manifest says what the code does.
func tokensAnswersEachStageAndRelaysTheSealedTokenOfOneNode(t *testing.T) {
r := newRig(t)
@@ -43,8 +44,8 @@ func tokensAnswersEachStageAndRelaysTheSealedTokenOfOneNode(t *testing.T) {
t.Errorf("row %+v lacks revision, expiry or ask, or relays a token for a list of all", row)
}
}
if !slices.Equal(stages, []string{"ace:asked", "g14:live", "novox:redeeming"}) || all.Fill.Keys != 6 || !all.Issuing.On {
t.Errorf("tokens %v, fill %+v, issuing %+v", stages, all.Fill, all.Issuing)
if !slices.Equal(stages, []string{"ace:asked", "g14:live", "novox:redeeming"}) || all.Fill.Keys != 6 {
t.Errorf("tokens %v, fill %+v", stages, all.Fill)
}
one, err := r.h.Tokens("g14", false)
if err != nil || len(one.Open) != 1 || one.Open[0].Sealed != "sealed-for-g14" {
@@ -59,11 +60,11 @@ func tokensAnswersEachStageAndRelaysTheSealedTokenOfOneNode(t *testing.T) {
}
}
}
// 1126/A2/AC1: with no lease key issuing is on, and says why.
// 1126/B9 (ADR 0590 clause 4): the seat is the only issuer since the controller's unassignment, so tokens says no
// gate of issuing any more: nothing turns it off.
on, _ := r.h.Tokens("", false)
mustContain(t, on.Issuing.Why, "unassignment")
if !on.Issuing.On {
t.Errorf("issuing is off with no lease key: %s", on.Issuing.Why)
if raw, _ := json.Marshal(on); strings.Contains(string(raw), `"issuing"`) {
t.Errorf("tokens still says a gate of issuing: %s", raw)
}
}
modules/mesh-enrolment/cmd/mesh-enrolment/tools.go
+1 -1
View File
@@ -106,7 +106,7 @@ func Tools(h *Holder) []stdio.Tool {
Run: func(a map[string]any) (any, error) { return h.Again(arg(a, "node")) }},
{Name: TokensTool,
Description: "What the seat holds (novox/hq ADR 0587 §8): every open enrolment, or one node's, with its stage (asked, live or " +
"redeeming), address, expiry, ask, revision and age; whether issuing is on and why; the bucket's fill. With a node at stage live, the token sealed to the operator key, asked of the vault at " +
"redeeming), address, expiry, ask, revision and age; the bucket's fill. With a node at stage live, the token sealed to the operator key, asked of the vault at " +
"this moment and relayed as the vault sealed it: the one route by which the operator gets a token, which this seat never " +
"reads and keeps nowhere. With history=true the ended enrolments its events say. A machine stuck at redeeming shows here " +
"with its age; revoke ends it. Read-only.",
modules/mesh-enrolment/go.mod
+2 -2
View File
@@ -3,8 +3,7 @@ module mesh-enrolment
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21
github.com/nats-io/nats-server/v2 v2.11.17
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791
github.com/nats-io/nats.go v1.54.0
)
@@ -14,6 +13,7 @@ require (
github.com/klauspost/compress v1.20.0 // indirect
github.com/minio/highwayhash v1.0.4 // indirect
github.com/nats-io/jwt/v2 v2.8.1 // indirect
github.com/nats-io/nats-server/v2 v2.11.17 // indirect
github.com/nats-io/nkeys v0.4.16 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
golang.org/x/crypto v0.57.0 // indirect
modules/mesh-enrolment/go.sum
+2 -6
View File
@@ -1,9 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101085317-790d1c82d027 h1:a33dsavoaqllh40eVICbhxi6Kero1EGSjt1V+2fa2Hk=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101085317-790d1c82d027/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101151945-b2a27ed5cec4 h1:xeuMBwOHNKrVUdGVg7Bdv5tkdueBvngwJFyk6n10rp0=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101151945-b2a27ed5cec4/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21 h1:4I4ayo0fZYP4PBfEX1sGDDd6arnw5IQJySJ3jcFobPM=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791 h1:7SYIDPMxbPBr8dMB6k9fYsgVt4ZyWVL41q4LMFsfAvs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
modules/mesh-grants/cmd/mesh-grants/asking.go
+2 -78
View File
@@ -2,77 +2,14 @@ package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// The three bus adapters for mesh-sdk go/asker on the part's own credential, as mesh-delivery-teller has them (copied: the
// SDK's asker reaches the bus through a bundle's runtime; moving the adapters into the SDK is a change of its own). What
// the account may do is the controller's composed grant for a module that uses the seat operator-channel: publish its own
// ask and condition, hear its own decided event, and read the router's record of its own asks.
const recordWithin = 30 * time.Second
// messengerAsks is the router's records bucket as the seat operator-channel's manifest declares it (its `records`), the
// one the controller composes this account's direct get on (mesh-sdk buscompose SeatTrafficOf): never `asks`, a bucket
// no grant names (issue 774). The SDK names no constant for it; mesh-mail and pacman keep their own the same way.
const messengerAsks = "messenger_asks"
func publishVia(ctx context.Context, js jetstream.JetStream) func(subject string, body any, id string) error {
return func(subject string, body any, id string) error {
data, err := json.Marshal(body)
if err != nil {
return err
}
var opts []jetstream.PublishOpt
if id != "" {
opts = append(opts, jetstream.WithMsgID(id))
}
_, err = js.Publish(ctx, subject, data, opts...)
return err
}
}
func recordVia(ctx context.Context, conn *nats.Conn, module string) func(id string) (*stdio.StateEntry, error) {
return func(id string) (*stdio.StateEntry, error) {
ctx, cancel := context.WithTimeout(ctx, recordWithin)
defer cancel()
msg, err := conn.RequestWithContext(ctx, "$JS.API.DIRECT.GET.KV_"+messengerAsks+".$KV."+messengerAsks+"."+module+"."+id, nil)
// No responders is the bucket out of reach for a moment, never "no record": read as none, a live ask would be
// closed by Reconcile a minute later and the operator's yes lost.
if errors.Is(err, nats.ErrNoResponders) {
return nil, fmt.Errorf("the router's record of %s cannot be reached: %w", id, err)
}
if err != nil {
return nil, err
}
if status := msg.Header.Get("Status"); status == "404" || len(msg.Data) == 0 {
return nil, nil
}
return &stdio.StateEntry{Key: id, Value: msg.Data}, nil
}
}
func decidedVia(conn *nats.Conn, module string, logf func(string, ...any)) func(pattern string, handle func(stdio.Envelope) error) error {
return func(pattern string, handle func(stdio.Envelope) error) error {
_, err := conn.Subscribe(asks.DecidedSubject(module), func(m *nats.Msg) {
if err := handle(stdio.Envelope{Key: pattern, Body: m.Data}); err != nil {
logf("a warrant could not be handled and is read again from the router's record at the next start: %v", err)
}
})
return err
}
}
// sayVia says a condition of the part on its own subject, as a module that uses the seat operator-channel may: the
// account consumers that differ, a grant missing, a user list held, a principal refused.
func sayVia(ctx context.Context, js jetstream.JetStream, module string) func(c asks.Condition) error {
return func(c asks.Condition) error {
subject, err := asks.ConditionSubject(module)
@@ -87,16 +24,3 @@ func sayVia(ctx context.Context, js jetstream.JetStream, module string) func(c a
return err
}
}
// seatSaying says the seat mesh-grants' events on its seat subject into EVENTS, as the module that holds it (novox/hq
// ADR 0582 clauses 3, 4, 6): lease-handed, lease-altered and removal-failed.
func seatSaying(ctx context.Context, js jetstream.JetStream) func(event string, body any) error {
return func(event string, body any) error {
data, err := json.Marshal(body)
if err != nil {
return err
}
_, err = js.Publish(ctx, "mesh.seat."+catalogue.GrantsSeat+".event."+event, data)
return err
}
}
modules/mesh-grants/cmd/mesh-grants/integration_test.go
+123 -237
View File
@@ -6,12 +6,11 @@ import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"slices"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
@@ -19,147 +18,21 @@ import (
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/asker"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-sdk/go/membership"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/seal"
"mesh-grants/internal/grants"
)
// standIns are the seats and tools the lease write asks, standing in on the real bus's side: node-service-manager on
// the controller's node (a unit's ActiveState, as the systemd module answers it), the bus module's expiry tool acting on
// the real lease bucket's stream (as nats-tools' SetBucketExpiry does), and the checks answering green.
type standIns struct {
ctx context.Context
js jetstream.JetStream
mu sync.Mutex
active string
calls []string
}
func (s *standIns) Call(key string, body any) (json.RawMessage, error) {
s.mu.Lock()
s.calls = append(s.calls, key)
s.mu.Unlock()
base, _, _ := strings.Cut(key, "@")
args, _ := body.(map[string]any)
var out any
switch base {
case grants.CallServiceStop:
s.active, out = "inactive", map[string]any{"ok": true}
case grants.CallServiceStart:
s.active, out = "active", map[string]any{"ok": true}
case grants.CallServiceStatus:
out = map[string]any{"ActiveState": s.active}
case grants.CallBucketExpiry:
now := fmt.Sprint(args["max_age"])
d, err := time.ParseDuration(strings.Replace(now, "none", "0s", 1))
if err != nil {
return nil, err
}
st, err := s.js.Stream(s.ctx, "KV_"+fmt.Sprint(args["bucket"]))
if err != nil {
return nil, err
}
cfg := st.CachedInfo().Config
cfg.MaxAge = d
if d > 0 && (cfg.Duplicates == 0 || cfg.Duplicates > d) {
cfg.Duplicates = d
}
if _, err := s.js.UpdateStream(s.ctx, cfg); err != nil {
return nil, err
}
out = map[string]any{"bucket": args["bucket"], "now": now, "changed": true}
case grants.CallIssue:
out = map[string]any{"status": "resolved"}
case grants.CallNodeTrusts:
out = map[string]any{"every": true, "not_yet": []string{}}
case grants.CallConfigCompare:
out = map[string]any{"source": "indexer", "count": 0}
case grants.CallUpgrade:
out = map[string]any{"empty": true, "splits": []any{}}
case grants.CallVaultStatus:
out = map[string]any{"ready": true, "minting": map[string]any{"on": true, "held_back": map[string]string{}}}
case grants.CallStreams:
out = map[string]any{"detail": []any{}}
case grants.CallUsers:
out = map[string]any{"users": []any{}}
case grants.CallRecordWarrant, grants.CallRecordHandAct:
out = map[string]any{}
default:
return nil, fmt.Errorf("no stand-in for %s", key)
}
return json.Marshal(out)
}
// memStore is the part's own state, in memory, with its time to live per key recorded.
type memStore struct {
mu sync.Mutex
vals map[string][]byte
revs map[string]uint64
}
func (m *memStore) Get(state, key string) ([]byte, uint64, bool, error) {
m.mu.Lock()
defer m.mu.Unlock()
v, ok := m.vals[state+"/"+key]
return v, m.revs[state+"/"+key], ok, nil
}
func (m *memStore) Put(state, key string, value []byte) (uint64, error) {
m.mu.Lock()
defer m.mu.Unlock()
m.vals[state+"/"+key] = append([]byte(nil), value...)
m.revs[state+"/"+key]++
return m.revs[state+"/"+key], nil
}
func (m *memStore) PutTTL(state, key string, value []byte, _ time.Duration) (uint64, error) {
return m.Put(state, key, value)
}
func (m *memStore) Keys(state string) ([]string, error) {
m.mu.Lock()
defer m.mu.Unlock()
var out []string
for k := range m.vals {
if s, key, _ := strings.Cut(k, "/"); s == state {
out = append(out, key)
}
}
return out, nil
}
func (m *memStore) Delete(state, key string) error {
m.mu.Lock()
defer m.mu.Unlock()
delete(m.vals, state+"/"+key)
return nil
}
type memObjects struct{}
func (memObjects) PutObject(string, []byte) error { return nil }
func (memObjects) DeleteObject(string) error { return nil }
// yesAsker answers every ask the part raises with the operator's yes, as the router's warrant hands it.
type yesAsker struct{ asked []string }
func (a *yesAsker) Ask(q asks.Ask) error { a.asked = append(a.asked, q.ID); return nil }
func (a *yesAsker) Where(string) (string, *asks.Warrant, error) { return "open", nil, nil }
func (a *yesAsker) Cancel(string) error { return nil }
func (a *yesAsker) Take(w asks.Warrant, act func(asks.Option, asks.Warrant) error) (asker.Handled, error) {
return asker.Acted, act(asks.Option{ID: w.Option}, w)
}
// Issue 774, 774/S23 (novox/hq ADR 0582 clauses 3, 5, "How it is checked"): mesh-grants' lease write and hand-back
// against the real bus, with a stand-in controller holding the lease in a bucket with an expiry (2 s standing in for 15
// s, so the test waits past it in seconds), and stand-in node-service-manager and bus tool seats.
func TestTheLeaseWriteOnTheBus(t *testing.T) {
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "lease")
// TestTheGrantsPartWhole starts the program the way the node does (its entry, run, with its environment and a broker
// file) against an embedded bus that holds no controller at all: no lease bucket, no controller placed (novox/hq ADR 0590,
// issue 1126). Given a module placed with an event it consumes, it publishes the module's membership in its own state and
// on no subject, makes the module's durable consumer on EVENTS, and writes the sealed list's key for the bus node to follow
// alone; and it creates no lease, nor any state of a lease write or a take-over.
func TestTheGrantsPartWhole(t *testing.T) {
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "grants")
if err != nil {
t.Fatal(err)
}
@@ -182,128 +55,141 @@ func TestTheLeaseWriteOnTheBus(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
const ttl = 2 * time.Second
was := grants.LeaseExpiry
grants.LeaseExpiry = ttl
t.Cleanup(func() { grants.LeaseExpiry = was })
lease, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: buscompose.LeaseBucket, History: 1, TTL: ttl})
if err != nil {
t.Fatal(err)
}
controller, _ := json.Marshal(buscompose.LeaseHolder{Instance: "controller@box-a pid 1", Host: "box-a", Epoch: 60})
if _, err := lease.Create(ctx, buscompose.ControllerLeaseKey, controller); err != nil {
t.Fatal(err)
}
bucket := func(state string) jetstream.KeyValue {
kvs := map[string]jetstream.KeyValue{}
for _, state := range grants.Reads {
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: bucketOf(state), History: 5})
if err != nil {
t.Fatal(err)
}
return kv
kvs[state] = kv
}
put := func(kv jetstream.KeyValue, key string, v any) {
for _, state := range grants.States {
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: grants.Module + "_" + state, History: 5})
if err != nil {
t.Fatal(err)
}
kvs[grants.Module+"."+state] = kv
}
for _, c := range []jetstream.StreamConfig{
{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}},
{Name: "CONDITIONS", Subjects: []string{"mesh.seat.operator-channel.accept.condition.>"}},
} {
if _, err := js.CreateStream(ctx, c); err != nil {
t.Fatal(err)
}
}
put := func(state, key string, v any) {
t.Helper()
raw, err := json.Marshal(v)
if err != nil {
t.Fatal(err)
}
if _, err := kv.Put(ctx, key, raw); err != nil {
if _, err := kvs[state].Put(ctx, key, raw); err != nil {
t.Fatal(err)
}
}
nats := `{"module":"nats","version":"1","claims":[{"name":"mesh-broker"}],"own-secrets":{"broker":"/var/lib/nats/broker"},` +
`"tools":["nats_bucket_expiry","nats_consumer_remove","nats_stream_remove"]}`
digest := delivery.ManifestDigest([]byte(nats))
put(bucket(grants.ReadManifests), digest, delivery.Manifest{V: delivery.Version, Module: "nats", JSON: nats})
put(bucket(grants.ReadModules), "nats", delivery.Module{V: delivery.Version, FP2: "src2:" + strings.Repeat("ab", 32), Manifest: digest,
Repository: "novox/mesh-catalog", Commit: "c"})
pl, _ := delivery.EncodePlacement(delivery.Placement{Node: "a", Modules: []delivery.PlacedModule{{Module: "nats"}, {Module: grants.Controller}}})
if _, err := bucket(grants.ReadPlacement).Put(ctx, "placement.a", pl); err != nil {
manifest := func(module, raw string) {
digest := delivery.ManifestDigest([]byte(raw))
put(grants.ReadManifests, digest, delivery.Manifest{V: delivery.Version, Module: module, JSON: raw})
put(grants.ReadModules, module, delivery.Module{V: delivery.Version, FP2: "src2:" + strings.Repeat("ab", 32), Manifest: digest,
Repository: "novox/x", Commit: "c"})
}
manifest("nats", `{"module":"nats","version":"1","claims":[{"name":"mesh-broker"}],"own-secrets":{"broker":"/var/lib/nats/broker"}}`)
manifest("mod-c", `{"module":"mod-c","version":"1","own-secrets":{"broker":"/var/lib/c/broker"},"consumes":["nats.done"]}`)
pl, _ := delivery.EncodePlacement(delivery.Placement{Node: "a", Modules: []delivery.PlacedModule{{Module: "nats"}, {Module: "mod-c"}}})
if _, err := kvs[grants.ReadPlacement].Put(ctx, "placement.a", pl); err != nil {
t.Fatal(err)
}
put(bucket(grants.ReadTakeover), health.TakeoverKey, health.Takeover{V: 1, Follow: health.FollowKeeper, By: "operator", Why: "taken"})
put(bucket(grants.ReadAccounts), buscompose.ControllerName, map[string]any{"hash": "$2a$11$controller", "generation": 1, "at": "2026-11-01T00:00:00Z"})
busKey, _ := seal.GenerateKeyPair()
have := bucket(grants.ReadHave)
loads := func(source, digest string) {
put(have, "a.nats", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK,
UserList: &delivery.UserListWritten{Source: source, Revision: 1, Digest: digest, Users: 1, Key: busKey.PublicKey}})
}
loads("a.nats.src2", "sha256:aa")
put(grants.ReadHave, "a.node", map[string]any{"v": 1})
put(grants.ReadHave, "a.nats", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK,
UserList: &delivery.UserListWritten{Source: delivery.GrantsUserListSource, Revision: 1, Digest: "sha256:aa", Users: 1,
Key: busKey.PublicKey, Follows: delivery.FollowGrants}})
seats := &standIns{ctx: ctx, js: js, active: "active"}
store := &memStore{vals: map[string][]byte{}, revs: map[string]uint64{}}
p := &grants.Part{Read: newBusReader(ctx, js), Call: seats, Store: store, Lease: &leaseHand{ctx: ctx, js: js},
Asker: &yesAsker{}, Objects: memObjects{}, Nonce: func() string { return "0001" }}
act := func(ask func(string, string) (any, error)) {
t.Helper()
if _, err := ask("operator@shanks", "issue 774's integration test"); err != nil {
t.Fatal(err)
}
p.Compose() // the process raises the ask
r, _ := p.LeaseState()
if r.Open == nil {
t.Fatalf("no act open: %+v", r)
}
if err := p.Warranted(asks.Option{ID: "yes"}, asks.Warrant{Ask: r.Open.Ask, Option: "yes", Outcome: asks.OutcomeChosen}); err != nil {
t.Fatal(err)
}
r, _ = p.LeaseState()
if r.Open == nil || r.Open.List == "" {
t.Fatalf("the act is not waiting on its list: %+v", r)
}
loads(delivery.GrantsUserListSource, r.Open.List) // the bus node loads it
p.Compose()
if r, _ := p.LeaseState(); r.Open != nil {
t.Fatalf("the act did not finish: %+v", r.Open)
}
// The bus module's read tools, as nats-tools answers them: the streams with their consumers as the bus holds them now.
answer := func(m *nats.Msg, result any) {
raw, _ := json.Marshal(map[string]any{"result": result, "node": "a"})
_ = m.Respond(raw)
}
expiry := func() time.Duration {
st, err := lease.Status(ctx)
if err != nil {
t.Fatal(err)
sub, err := nc.Subscribe("mesh.mod.nats.>", func(m *nats.Msg) {
switch {
case strings.Contains(m.Subject, ".nats_users"):
answer(m, map[string]any{"users": []any{}})
case strings.Contains(m.Subject, ".nats_streams"):
var args struct{ Stream string }
_ = json.Unmarshal(m.Data, &args)
detail := []any{}
if st, err := js.Stream(ctx, args.Stream); err == nil {
consumers := []any{}
names := st.ConsumerNames(ctx)
for name := range names.Name() {
c, _ := st.Consumer(ctx, name)
cfg := c.CachedInfo().Config
consumers = append(consumers, map[string]any{"name": name, "filters": cfg.FilterSubjects, "max_deliver": cfg.MaxDeliver})
}
detail = append(detail, map[string]any{"name": args.Stream, "consumer_detail": consumers})
}
answer(m, map[string]any{"detail": detail})
default:
answer(m, map[string]any{})
}
return st.TTL()
}
// Issue 774: the operator's answer is read from the router's own bucket, the one the controller grants (the seat's
// records, messenger_asks), as the process reads it.
put(bucket(messengerAsks), "mesh-grants.lease-write-0001", map[string]any{"status": "decided"})
if rec, err := recordVia(ctx, nc, "mesh-grants")("lease-write-0001"); err != nil || rec == nil {
t.Fatalf("the router's record of the ask: %v, %v; want it read", rec, err)
}
// 774/S23/AC1.
act(p.LeaseWrite)
e, err := lease.Get(ctx, buscompose.ControllerLeaseKey)
})
if err != nil {
t.Fatal(err)
}
if away, says := buscompose.LeaseHandedAway(e.Value(), true, nil); !away || !strings.Contains(string(e.Value()), `"handed-to":"mesh-grants"`) {
t.Fatalf("the lease after the write: %s (%s)", e.Value(), says)
}
if expiry() != 0 || seats.active != "inactive" {
t.Fatalf("the lease's expiry is %s, the controller %s", expiry(), seats.active)
}
// Issue 774: the stop goes to the node placement places the controller on, not the lease's host.
if !slices.Contains(seats.calls, grants.CallServiceStop+"@a") || slices.Contains(seats.calls, grants.CallServiceStop+"@box-a") {
t.Fatalf("the stops asked: %v, want one on a", seats.calls)
}
if _, err := lease.Create(ctx, buscompose.ControllerLeaseKey, controller); !errors.Is(err, jetstream.ErrKeyExists) {
t.Fatalf("the controller's retake: %v, want refused", err)
}
time.Sleep(ttl + time.Second)
if e, err := lease.Get(ctx, buscompose.ControllerLeaseKey); err != nil {
t.Fatalf("the lease is gone past the old expiry: %v", err)
} else if away, _ := buscompose.LeaseHandedAway(e.Value(), true, nil); !away {
t.Fatalf("the lease past the old expiry: %s", e.Value())
t.Cleanup(func() { _ = sub.Unsubscribe() })
// A membership is never said on its subject any more (issue 1126).
var said atomic.Int32
if _, err := nc.Subscribe("mesh.assignment.>", func(*nats.Msg) { said.Add(1) }); err != nil {
t.Fatal(err)
}
// 774/S23/AC2.
act(p.LeaseHandBack)
if expiry() != ttl || seats.active != "active" {
t.Fatalf("after the hand-back the expiry is %s, the controller %s", expiry(), seats.active)
broker := filepath.Join(dir, "broker")
cred, _ := json.Marshal(process.Account{URL: srv.ClientURL(), Node: "a", Module: grants.Module, User: "a." + grants.Module, Password: "pw"})
if err := os.WriteFile(broker, cred, 0o600); err != nil {
t.Fatal(err)
}
if _, err := lease.Create(ctx, buscompose.ControllerLeaseKey, controller); err != nil {
t.Fatalf("the controller cannot take the lease back: %v", err)
env := map[string]string{"MESH_BROKER_FILE": broker}
runCtx, stop := context.WithCancel(ctx)
done := make(chan error, 1)
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
t.Cleanup(func() { stop(); <-done })
until := func(what string, ok func() bool) {
t.Helper()
for deadline := time.Now().Add(30 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if ok() {
return
}
}
t.Fatalf("never saw: %s", what)
}
until("a.mod-c's membership in the part's own state memberships", func() bool {
e, err := kvs[grants.Module+"."+grants.StateMemberships].Get(ctx, "a.mod-c")
if err != nil {
return false
}
m, err := membership.DecodeGrantsMembership(e.Value())
return err == nil && m.Module == "mod-c"
})
until("mod-c's durable consumer on EVENTS, made by the part", func() bool {
_, err := js.Consumer(ctx, "EVENTS", "a_mod-c")
return err == nil
})
until("the sealed list's key, followed by the bus node alone", func() bool {
e, err := kvs[grants.Module+"."+grants.StateUserList].Get(ctx, grants.KeySealed)
if err != nil {
return false
}
g, err := delivery.DecodeGrantsUserList(e.Value())
return err == nil && g.Follow == delivery.FollowGrants
})
if n := said.Load(); n != 0 {
t.Errorf("%d memberships were said on their subject", n)
}
for _, bucket := range []string{"mesh-controller_lease", grants.Module + "_lease", grants.Module + "_takeover"} {
if _, err := js.KeyValue(ctx, bucket); !errors.Is(err, jetstream.ErrBucketNotFound) {
t.Errorf("the bucket %s: %v, want none", bucket, err)
}
}
}
modules/mesh-grants/cmd/mesh-grants/lease.go
-79
View File
@@ -1,79 +0,0 @@
package main
import (
"context"
"encoding/json"
"fmt"
"sync"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/buscompose"
)
// leaseHand is the process's hand on the controller's lease (novox/hq ADR 0582 clause 3): its key holder, which the
// lease write's cut grants the part alone, and the bucket's expiry, read from the bucket's own stream.
type leaseHand struct {
ctx context.Context
js jetstream.JetStream
mu sync.Mutex
kv jetstream.KeyValue
}
func (l *leaseHand) bucket() (jetstream.KeyValue, error) {
l.mu.Lock()
defer l.mu.Unlock()
if l.kv != nil {
return l.kv, nil
}
ctx, cancel := context.WithTimeout(l.ctx, requestWithin)
defer cancel()
kv, err := l.js.KeyValue(ctx, buscompose.LeaseBucket)
if err != nil {
return nil, fmt.Errorf("the lease's bucket cannot be opened: %w", err)
}
l.kv = kv
return kv, nil
}
func (l *leaseHand) PutHolder(value []byte) error {
kv, err := l.bucket()
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(l.ctx, requestWithin)
defer cancel()
_, err = kv.Put(ctx, buscompose.ControllerLeaseKey, value)
return err
}
func (l *leaseHand) DeleteHolder() error {
kv, err := l.bucket()
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(l.ctx, requestWithin)
defer cancel()
return kv.Delete(ctx, buscompose.ControllerLeaseKey)
}
func (l *leaseHand) Expiry() (time.Duration, error) {
kv, err := l.bucket()
if err != nil {
return 0, err
}
ctx, cancel := context.WithTimeout(l.ctx, requestWithin)
defer cancel()
st, err := kv.Status(ctx)
if err != nil {
return 0, err
}
return st.TTL(), nil
}
// PutTTL writes a key of the part's own state with its own time to live (the state lease).
func (s ownStore) PutTTL(state, key string, value []byte, ttl time.Duration) (uint64, error) {
return s.c.State(state).RenewTTL(key, json.RawMessage(value), ttl)
}
modules/mesh-grants/cmd/mesh-grants/main.go
+10 -26
View File
@@ -4,21 +4,17 @@
// list and each manifest's buckets, logs and streams with the bus composer the controller uses (mesh-sdk buscompose).
//
// **Memberships are published for real** (the operator, 2026-10-25 22:31): since release 6 nobody publishes them, so a
// module placed since cannot serve its seat or reach a new state. Each is published in the part's own state memberships,
// and said on its subject, only when it differs from the one last published there. **A bucket, log or stream a manifest declares that the bus lacks is
// module placed since cannot serve its seat or reach a new state. Each is published in the part's own state memberships
// only when it differs from the one last published there. **A bucket, log or stream a manifest declares that the bus lacks is
// created** (the operator, 2026-10-26 08:54; issue 1790), once per manifest, with the manifest's limits, on the event
// that brought the manifest; one the bus holds is never changed. **The user list is written sealed to the bus node**
// (ADR 0547 clause 7) as an object of the part's own bucket named in `user-list.sealed`, which the bus node follows only
// from take-over. **The rest of the assertions and the grants stay in shadow** until
// the operator's take-over, asked on their channel and run on their yes after its refusals: they are composed, compared with what the bus loaded
// and holds (through the bus module's own read tools) and kept in the part's own state, and take no effect.
// (ADR 0547 clause 7) as an object of the part's own bucket named in `user-list.sealed`, which the bus node follows
// alone since the controller ended by its unassignment (novox/hq ADR 0590, issue 1126). **The grants** the assigner
// reads are written, and **the account consumers** made; a stream the bus holds is never changed, a difference only said.
//
// It holds no timer, and never exits for a refused read, write or publish: a refusal is said and tried again at the next
// event.
//
// **The seat mesh-grants** (novox/hq ADR 0582 clause 1, issue 774): the part holds it for the lease write, serving
// lease-write and lease-hand-back beside its own grants_* tools, and says lease-handed, lease-altered and removal-failed.
//
// Started by the tool runner, with no credential of its own, it serves those tools (a process never serves, ADR 0436).
package main
@@ -38,7 +34,6 @@ import (
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/announce"
"git.novox.be/novox/mesh-sdk/go/asker"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-grants/internal/grants"
@@ -110,21 +105,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
bus := newBusReader(ctx, js)
f.Part = &grants.Part{Read: bus, Call: conn, Store: ownStore{conn}, Emit: emitting{conn},
Publish: memberships{nc: nc}, Create: creating{ctx: ctx, js: js}, Consumers: creating{ctx: ctx, js: js},
Create: creating{ctx: ctx, js: js}, Consumers: creating{ctx: ctx, js: js},
Objects: sealedObjects{ctx: ctx, js: js}, Logf: logger.Printf,
Bus: grants.BusAddress{URL: account.URL, Fingerprint: account.Fingerprint}}
f.Watch = bus
// The operator's word on take-over and hand-back is their yes on their channel: the part asks it itself, and acts on
// the warrant once (mesh-sdk asker), after its refusals.
client := &asker.Client{Module: account.Module, State: conn.State(grants.StateTakeover), Publish: publishVia(ctx, js),
Record: recordVia(ctx, nc, account.Module), Subscribe: decidedVia(nc, account.Module, logger.Printf)}
f.Part.Asker, f.Part.Say = client, sayVia(ctx, js, account.Module)
// The lease write's hand on the lease, and its seat's events (ADR 0582 clauses 3, 4, 6).
f.Part.Lease, f.Part.SaySeat = &leaseHand{ctx: ctx, js: js}, seatSaying(ctx, js)
f.Part.Forget = func(id string) error { return conn.State(grants.StateTakeover).Delete("ask." + id) }
if err := client.Listen(f.Part.Warranted, f.Part.Said); err != nil {
logger.Printf("the operator's answers cannot be heard yet, so take-over and hand-back wait: %v", err)
}
f.Part.Say = sayVia(ctx, js, account.Module)
f.Asked = func(wake func()) error {
return conn.State(grants.StateUserList).Watch(grants.KeyAsked, func(c stdio.StateChange) error {
if !c.Current {
@@ -133,9 +118,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return nil
})
}
// The controller makes this module's durable consumer on EVENTS from the manifest's consumes (filtered on the two
// condition subjects and the vault's account-issued; ADR 0561 §6 gives that to the part from the lease write); it is
// pulled and every message acknowledged, so its pending never grows. Where it is not there yet the pass says so and
// The part makes this module's own durable consumer on EVENTS from the manifest's consumes (filtered on the two
// condition subjects and the vault's account-issued; ADR 0561 §6), as it makes every module's; it is pulled and every message acknowledged, so its pending never grows. Where it is not there yet the pass says so and
// hears again at the next change. The vault's bus-accounts is watched beside it, so a hash is composed either way.
f.Hear = func(handle func(string, []byte)) error {
cons, err := js.Consumer(ctx, "EVENTS", account.Node+"_"+account.Module)
@@ -148,7 +132,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}, jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) }))
return err
}
logger.Printf("started on the account %s: publishing memberships and creating the declared buckets, logs and streams the bus lacks; and writing the user list sealed to the bus node, which the bus node follows from take-over; grants and any change to a held stream in shadow", account.User)
logger.Printf("started on the account %s: publishing memberships, writing grants and the user list sealed to the bus node, which it follows, and creating the account consumers and the declared buckets, logs and streams the bus lacks; a change to a held stream is only said", account.User)
f.Run(ctx)
return nil
}
modules/mesh-grants/cmd/mesh-grants/publish.go
-24
View File
@@ -1,24 +0,0 @@
package main
import (
"fmt"
"github.com/nats-io/nats.go"
"git.novox.be/novox/mesh-sdk/go/membership"
)
// memberships says each membership the part wrote to its state on its subject, `mesh.assignment.<node>.<module>`, as a
// plain message: a notice for a runtime that follows the subject alone, which the assignments stream also keeps while it
// exists. It reads nothing there and asks no stream for an acknowledgement, so the stream's removal at the lease write
// (research 072 row 33) stops no membership (issue 774). The one place the part publishes a membership on the bus.
type memberships struct {
nc *nats.Conn
}
func (m memberships) PublishMembership(node, module string, body []byte) error {
if err := m.nc.Publish(membership.Subject(node, module), body); err != nil {
return fmt.Errorf("the membership of %s on %s: %w", module, node, err)
}
return nil
}
modules/mesh-grants/cmd/mesh-grants/tools.go
+2 -67
View File
@@ -6,7 +6,6 @@ import (
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"mesh-grants/internal/grants"
)
@@ -85,7 +84,7 @@ func toolsOf(p *grants.Part) []stdio.Tool {
Run: func(a map[string]any) (any, error) {
return p.MembershipOf(grants.Answer(a, "node"), grants.Answer(a, "module"))
}},
{Name: grants.ToolUserList, Description: "The bus's user list in shadow (ADR 0547 clauses 6 and 8): the list the grants part " +
{Name: grants.ToolUserList, Description: "The bus's user list (ADR 0547 clauses 6 and 8): the list the grants part " +
"composed (digest, revision, users), what the bus node says it loaded (source, revision, digest, users, error), and how " +
"they differ: users added, users dropped (and of those, gone from placement), each user's grant differences, the " +
"assignments left out, every input the bus does not carry, named; and per user whether the vault holds its hash " +
@@ -98,9 +97,7 @@ func toolsOf(p *grants.Part) []stdio.Tool {
"state (one revision when an input changed, none when not), saying user-list-changed and membership-changed. Answers " +
"the ask and the last pass; grants_user_list shows the pass that answers it. A membership is published unless it would " +
"leave a seat a placed module claims with no holder, or drop a capability its holder still asks for (verified-sender), " +
"which stays a shadow difference, named. The user list, assertions and " +
"grants stay in shadow. It cannot ask the controller to write a module's grant again: the controller serves no such " +
"verb, and the answer says so.",
"which stays a shadow difference, named.",
Run: func(a map[string]any) (any, error) { return p.AskCompose(stdio.CallerOf(a)) }},
{Name: grants.ToolAssert, Description: "One module's buckets, logs and streams (ADR 0547 clause 6), composed from its manifest at " +
"the indexer's trunk with their limits, beside what the bus holds (through the bus module's nats_streams). When the " +
@@ -109,21 +106,6 @@ func toolsOf(p *grants.Part) []stdio.Tool {
"changed, a difference is only said. Answers the comparison and the ask; call again to see what the bus holds.",
Input: schema(map[string]string{"module": "the module"}, "module"),
Run: func(a map[string]any) (any, error) { return p.AssertOne(grants.Answer(a, "module")) }},
{Name: grants.ToolTakeOver, Description: "Ask the operator to switch the bus node to the grants part's list (ADR 0547 clause 8). " +
"The operator's word is their yes on their channel (a warrant), never a caller's name: this tool only asks, once at a time. " +
"After the yes, the switch is recorded and takes effect when the bus node loads the part's list; it is refused while the loaded list could not be read, a user differs between the lists " +
"(an enrolment user or a person the part cannot compose among them), or the vault holds no hash for a user, and the " +
"refusal is said to the operator naming the users. Every act is recorded in the part's state takeover.",
Input: schema(map[string]string{"why": "why, kept with the act"}, "why"),
Run: func(a map[string]any) (any, error) {
return p.TakeOver(stdio.CallerOf(a), grants.Answer(a, "why"))
}},
{Name: grants.ToolHandBack, Description: "Gone: the controller ended by its unassignment (issue 1126), so nothing writes its user list and the grants " +
"part's list is the only one; it says so and asks nothing (ADR 0547 §11).",
Input: schema(map[string]string{"why": "why, kept with the act"}, "why"),
Run: func(a map[string]any) (any, error) {
return p.HandBack(stdio.CallerOf(a), grants.Answer(a, "why"))
}},
{Name: grants.ToolAccounts, Description: "The bus accounts the mesh wants (novox/hq ADR 0561 §2, §9): every account the vault " +
"mints that placement composes (a module's, a process's, a machine's provisioner client's), as the grants part last " +
"wrote it to its state accounts, which the vault alone reads: user, kind, node, module, process, the bus's address and " +
@@ -160,56 +142,9 @@ func toolsOf(p *grants.Part) []stdio.Tool {
}
return p.AcceptHashes(stdio.CallerOf(a), users, grants.Answer(a, "why"))
}},
{Name: grants.ToolTakeOverState, Description: "Where the switch stands: the list the bus node follows, the ask open now, the request, and " +
"every act with what became of it (asked, switched, declined, expired, refused and why).",
Input: schema(nil),
Run: func(map[string]any) (any, error) { return p.TakeOverState() }},
{Name: grants.ToolLeaseCheck, Description: "The controller's lease and the controller's hand-over, in one call from bus state (issue 774, " +
"design 47 L3; novox/hq ADR 0547 §11, ADR 0561 §3, ADR 0571 §13): who holds the controller lease now, since when, and what " +
"the holder and the parts watching the lease do because of it; what still depends on the controller: every module whose " +
"manifest at the indexer's trunk reads a mesh-controller state, invokes a mesh-controller verb or tool, consumes or waits on " +
"its events, with which one and the record that keeps it until the lease write (the named keep list), the unkept ones by module; " +
"the controller's remaining live duties as the bus shows them: each of its streams and buckets someone consumes (with the " +
"durable consumers by name), and its standing assignments; and a dry run of the lease write's check (the take-over check that " +
"hands the lease to a holder that is no controller): whether it would pass now, at the write with only the three kept to the " +
"write and before the write's delivery with the whole list, each with its exact reasons, and the write's other conditions " +
"with the tool that answers each. Asks the controller nothing; writes nothing. Read-only.",
Input: schema(nil),
Run: func(map[string]any) (any, error) { return p.LeaseCheck() }},
// The seat's two verbs (ADR 0582 clauses 1, 5): asked of the operator by the part's process.
{Name: grants.SeatTool(grants.ActLeaseWrite), Description: seatVerb(grants.ActLeaseWrite).Description,
Input: seatVerb(grants.ActLeaseWrite).Input,
Run: func(a map[string]any) (any, error) {
return p.LeaseWrite(stdio.CallerOf(a), grants.Answer(a, "why"))
}},
{Name: grants.SeatTool(grants.ActLeaseHandBack), Description: seatVerb(grants.ActLeaseHandBack).Description,
Input: seatVerb(grants.ActLeaseHandBack).Input,
Run: func(a map[string]any) (any, error) {
return p.LeaseHandBack(stdio.CallerOf(a), grants.Answer(a, "why"))
}},
{Name: grants.ToolRemovalDue, Description: "Write the lease write's removal-due again with a short time to live, so " +
"its expiry marker starts the day-later removal of the controller's consumer of DEAD_LETTER_NOTICES and the " +
"stream ASSIGNMENTS (novox/hq ADR 0582 clause 6): the recovery for a marker missed while the part was down, the " +
"removal still started by the marker and nothing else. Refused unless the lease is handed to mesh-grants and the " +
"removal has not run. Recorded as a hand act.",
Input: schema(map[string]string{"why": "why, kept with the hand act", "ttl": "the time to live, e.g. 10s (1s to 24h)"}, "why", "ttl"),
Run: func(a map[string]any) (any, error) {
return p.RemovalDue(stdio.CallerOf(a), grants.Answer(a, "why"), grants.Answer(a, "ttl"))
}},
}
}
// seatVerb is the seat mesh-grants' verb as mesh-sdk declares it.
func seatVerb(name string) catalogue.Verb {
seat, _ := catalogue.SeatNamed(catalogue.GrantsSeat)
for _, v := range seat.Serves {
if v.Name == name {
return v
}
}
return catalogue.Verb{Name: name}
}
// serveTools serves the tools to the runtime until it stops.
func serveTools() error {
p := &grants.Part{Read: runtimeReader{}, Call: runtimeBus{}, Store: runtimeStore{}, Emit: runtimeBus{}}
modules/mesh-grants/go.mod
+1 -2
View File
@@ -3,7 +3,7 @@ module mesh-grants
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
@@ -16,7 +16,6 @@ require (
github.com/nats-io/jwt/v2 v2.8.1 // indirect
github.com/nats-io/nkeys v0.4.16 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
go.uber.org/automaxprocs v1.6.0 // indirect
golang.org/x/crypto v0.57.0 // indirect
golang.org/x/sys v0.48.0 // indirect
golang.org/x/time v0.15.0 // indirect
modules/mesh-grants/go.sum
+2 -8
View File
@@ -1,9 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101151945-b2a27ed5cec4 h1:xeuMBwOHNKrVUdGVg7Bdv5tkdueBvngwJFyk6n10rp0=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101151945-b2a27ed5cec4/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c h1:/YCxh+NYfqry3PxZYY1nQXaccupobxpsFt8c9BYuGBg=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21 h1:4I4ayo0fZYP4PBfEX1sGDDd6arnw5IQJySJ3jcFobPM=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791 h1:7SYIDPMxbPBr8dMB6k9fYsgVt4ZyWVL41q4LMFsfAvs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -22,8 +18,6 @@ github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs=
go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
modules/mesh-grants/grants_test.go
+237 -542
View File
File diff suppressed because it is too large Load Diff
modules/mesh-grants/internal/grants/accounts.go
+1 -1
View File
@@ -139,7 +139,7 @@ func (p *Part) Accounts(user string) (any, error) {
answer["last_pass"] = res.Accounts
}
// And the durable consumers the accounts need, as the last pass found them: missing and differing ones are read
// here in shadow too, where no condition is raised (ADR 0561 §6, §9).
// here too (ADR 0561 §6, §9).
if res.Consumers != nil {
answer["consumers"] = res.Consumers
}
modules/mesh-grants/internal/grants/assertions.go
+2 -2
View File
@@ -50,11 +50,11 @@ type Assertion struct {
}
// ShadowAsserts is what an assertion says where nothing may be created: in the tools, which ask the part's process.
const ShadowAsserts = "in shadow: composed and compared, nothing asserted (ADR 0547 clause 5 asserts from take-over)"
const ShadowAsserts = "in the tools: composed and compared, nothing asserted (the part's process creates what the bus lacks)"
// CreatesMissing is what an assertion of the part's process says (the operator, 2026-10-26 08:54; issue 1790).
const CreatesMissing = "a bucket, log or stream the manifest declares and the bus lacks is created with the manifest's limits, once per " +
"manifest; one the bus holds is never changed or deleted, a difference is only said (ADR 0547 clause 5 asserts the rest from take-over)"
"manifest; one the bus holds is never changed or deleted, a difference is only said"
// Creator creates one stream the bus does not hold, with the configuration given, and never updates or deletes one: the
// part's process alone, whose account is granted the create of each declared stream by name (the operator, 2026-10-26
modules/mesh-grants/internal/grants/consumers.go
+64 -15
View File
@@ -14,18 +14,10 @@ import (
"git.novox.be/novox/mesh-sdk/go/buscompose"
)
// **The durable consumers an account needs** (novox/hq ADR 0561 §6, issue 774). The controller creates a module's
// consumer on EVENTS and each process's on its work queue when it issues the account. From the write that hands its
// lease away the grants part creates them, with the configuration the bus composer derives (buscompose.AccountConsumers),
// only where the bus lacks one; one the bus holds is never changed or deleted, a difference is only said. Before that
// write they are compared in shadow alone, so two parts never create one consumer.
// ReadLease is the controller's lease as the part reads it (buscompose.NamedReaders names the vault and the part), and
// LeaseKey its one key.
const (
ReadLease = buscompose.ControllerLease
LeaseKey = buscompose.ControllerLeaseKey
)
// **The durable consumers an account needs** (novox/hq ADR 0561 §6, issue 774): a module's consumer on EVENTS and each
// process's on its work queue. Since the controller ended by its unassignment (ADR 0590, issue 1126) the grants part
// alone creates them, with the configuration the bus composer derives (buscompose.AccountConsumers), only where the bus
// lacks one; one the bus holds is never changed or deleted by a pass, a difference is only said.
// ConsumerCreator creates one durable consumer the bus does not hold: the part's process alone, whose account is granted
// the create of each account's consumer by stream and name. One the bus holds answers ErrHeld.
@@ -47,6 +39,9 @@ type ConsumersState struct {
Wanted int `json:"wanted"`
Missing []string `json:"missing,omitempty"`
Differ []string `json:"differ,omitempty"`
// Behind are those held more than BehindBound messages behind their stream's head at this pass and the one before,
// each with how far.
Behind []string `json:"behind,omitempty"`
Created []string `json:"created,omitempty"`
// Matched is a consumer brought to the composed configuration on a person's ask, or why not.
Matched []string `json:"matched,omitempty"`
@@ -60,6 +55,7 @@ type heldConsumer struct {
Name string `json:"name"`
Filters []string `json:"filters"`
MaxDeliver int `json:"max_deliver"`
Pending int64 `json:"pending"`
}
// heldConsumers reads the consumers the bus holds on stream, by name.
@@ -89,9 +85,8 @@ func heldConsumers(c Caller, stream string) (map[string]heldConsumer, error) {
return out, nil
}
// ConsumersOn is why the part creates and matches the account consumers with no lease read: the controller ended by its
// unassignment, never by the lease write, so its lease names no holder for good and the part is their only maker
// (issue 1126; mesh-sdk's buscompose.LeaseHandedAway answers false for an absent key, so it is no longer asked here).
// ConsumersOn is why the part creates and matches the account consumers: the controller ended by its unassignment, so
// the part is their only maker (novox/hq ADR 0590, issue 1126).
const ConsumersOn = "the grants part makes and matches the account consumers: the controller ended by its unassignment (issue 1126)"
// consumers compares the account consumers the records compose with those the bus holds, and creates the missing ones
@@ -107,6 +102,8 @@ func (p *Part) consumers(res *Result, recs buscompose.Records, match string) {
res.Consumers = st
wanted := buscompose.AccountConsumers(users)
st.Wanted = len(wanted)
seen := map[string]bool{}
defer func() { p.behindSeen = seen }()
byStream := map[string][]buscompose.Consumer{}
for _, mc := range wanted {
byStream[mc.Consumer.Stream] = append(byStream[mc.Consumer.Stream], mc.Consumer)
@@ -121,6 +118,12 @@ func (p *Part) consumers(res *Result, recs buscompose.Records, match string) {
key := stream + "." + c.Name
h, ok := held[c.Name]
if ok {
if h.Pending > BehindBound {
if p.behindSeen[key] {
st.Behind = append(st.Behind, fmt.Sprintf("%s: %d pending", key, h.Pending))
}
seen[key] = true
}
why := consumerDiffers(c, h)
if why != "" && key == match && st.On && p.Consumers != nil {
if err := p.Consumers.MatchConsumer(stream, buscompose.ConsumerConfig(c)); err != nil {
@@ -151,6 +154,7 @@ func (p *Part) consumers(res *Result, recs buscompose.Records, match string) {
}
}
p.sayDiffer(st)
p.sayBehind(st)
}
// sayDiffer raises the warning condition mesh-grants.consumer-differs while an account consumer the part owns (from the
@@ -236,3 +240,48 @@ func consumerDiffers(c buscompose.Consumer, h heldConsumer) string {
sort.Strings(why)
return strings.Join(why, "; ")
}
// BehindBound is how many messages behind its stream's head a consumer may be before it is said far behind: the bound
// mesh-self-check's bus-consumer probe judged by, which ended with the controller (issue 1126).
const BehindBound = 1000
// ConsumerBehindCondition is the key of the warning raised while an account consumer stays far behind its stream's head.
const ConsumerBehindCondition = "mesh-grants.consumer-behind"
// sayBehind raises the warning condition mesh-grants.consumer-behind while an account consumer the part owns is more
// than BehindBound messages behind its stream's head at two passes running (one look at a burst is not a consumer that
// stays behind, novox/hq issue 277), naming each with how far, and clears it when none is. Said once per set, again
// when it changes.
func (p *Part) sayBehind(st *ConsumersState) {
if p.Say == nil {
return
}
// The consumers alone decide whether it is said again, never how far each is behind, which changes every pass: the
// counts stay in the last pass's consumers.behind (its state user-list, key pass), and the condition is not said again
// for them.
var names []string
for _, b := range st.Behind {
name, _, _ := strings.Cut(b, ": ")
names = append(names, name)
}
why := strings.Join(names, ", ")
p.holdMu.Lock()
defer p.holdMu.Unlock()
if p.behindKnown && why == p.behindSaid {
return
}
c := asks.Condition{Key: ConsumerBehindCondition, Subject: asks.Scope{Scope: "module", ID: "mesh-grants"}}
if why == "" {
c.State = asks.StateCleared
} else {
c.Severity, c.State = asks.SeverityWarning, asks.StateRaised
c.Summary = fmt.Sprintf("%d durable consumers are more than %d messages behind their stream's head", len(st.Behind), BehindBound)
c.Headline, c.Needs = "Messages pile up for a listener", "a look at the module that reads them"
c.Explanation = asks.NeedsYou + "a look. These consumers stayed far behind their stream's head at two passes " +
"running, so what is sent to their modules waits unread (how far each is, in the grants part's last pass): " + why
c.Resolved = "every account consumer is back within the bound"
}
if p.Say(c) == nil {
p.behindSaid, p.behindKnown = why, true
}
}
modules/mesh-grants/internal/grants/follow.go
+10 -71
View File
@@ -6,9 +6,7 @@ import (
"strings"
"sync"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
)
// Watcher watches another part's state: changed hears each change after what stands, ended hears the watch end. A watch
@@ -21,14 +19,10 @@ type Watcher interface {
// config values, the node agents' haves (a machine's first have, and the bus node's loaded list) and the assigner's
// wants (the manifest each machine is wanted to run), and the vault's hashes of the bus accounts.
//
// And its own state lease, whose key removal-due's expiry marker starts the day-later removal (ADR 0582 clause 6), and
// the seat mesh-enrolment's keys (ADR 0587 §4, §5): a key at live composes enrol.<node>, and its end, by the holder or by
// And the seat mesh-enrolment's keys (ADR 0587 §4, §5): a key at live composes enrol.<node>, and its end, by the holder or by
// the bus's expiry, is a change of the key that takes the user out; no event is needed beside the watch.
var Followed = []string{ReadModules, ReadResolved, ReadHave, ReadWants, ReadAccounts, ReadPlacement, ReadLease, ReadTakeover,
ReadEnrolments, ReadNodeLogins, OwnLease}
// OwnLease is the part's own state lease as the follower watches it, `<module>.<state>`.
const OwnLease = Module + "." + StateLease
var Followed = []string{ReadModules, ReadResolved, ReadHave, ReadWants, ReadAccounts, ReadPlacement, ReadEnrolments,
ReadNodeLogins}
// Follower composes again on every change it hears, never on a timer. It never stops for a refusal: a watch it cannot
// open is said, composed without, and opened again at the next wake (any change it does hear, or a reconnect).
@@ -38,8 +32,8 @@ type Follower struct {
Logf func(string, ...any)
// Asked watches the part's own ask to compose now (a tool's compose-now), waking at each; nil watches none.
Asked func(wake func()) error
// Hear hears the controller seat's condition events and the vault's account-issued (Consumes) and hands each, by event
// name, with its body, to handle; nil hears none. A condition of the kind root-not-free raised or cleared wakes a pass:
// Hear hears the keeper's condition events and the vault's account-issued (Consumes) and hands each, by event name,
// with its body, to handle; nil hears none. A condition of the kind root-not-free raised or cleared wakes a pass:
// a machine's root-free word changed; and so does an account the vault minted (ADR 0561 §1), whose hash the list carries.
Hear func(handle func(event string, body []byte)) error
@@ -56,21 +50,6 @@ type Follower struct {
held string
notAsserted string
refusedMans string
// leaseOn is whether the lease was handed away at its last change heard, once leaseSeen.
leaseOn, leaseSeen bool
// follows is the seat whose condition events the part hears, as the take-over key names it: the controller's until
// the operator's take-over (an absent key, or one not readable, follows the controller).
follows string
}
// Follows is the seat whose condition events the part hears now.
func (f *Follower) Follows() string {
f.mu.Lock()
defer f.mu.Unlock()
if f.follows == "" {
return "mesh-controller"
}
return f.follows
}
func (f *Follower) init() {
@@ -110,43 +89,6 @@ func (f *Follower) logf(format string, args ...any) {
// heard says whether a change composes again: every change but a have's, and of the haves a machine's first and the
// bus module's loaded list moving.
func (f *Follower) heard(state, key string, value []byte) bool {
if state == ReadTakeover {
// Which seat's condition events are heard: no pass, since nothing composed reads it.
if key != health.TakeoverKey {
return false
}
t, err := health.DecodeTakeover(value, len(value) > 0)
if err != nil {
f.logf("the take-over key cannot be read (%v): the controller's condition events are heard", err)
t = health.Takeover{Follow: health.FollowController}
}
f.mu.Lock()
was := f.follows
f.follows = t.Seat()
f.mu.Unlock()
if was != t.Seat() {
f.logf("root-free words are heard from %s's condition events (the take-over key follows %s)", t.Seat(), t.Follow)
}
return false
}
if state == OwnLease {
// The marker of removal-due (its value gone: expired, or deleted by a hand-back, which the removal tells apart).
if key == KeyRemovalDue && value == nil && f.Part != nil {
f.Part.RemovalDueHeard()
return true
}
return false
}
if state == ReadLease {
// Renewed every few seconds by its holder: only the write that hands it away, or back, composes again (ADR 0561 §6).
// Once handed away nobody renews it, so any change is heard: it may be the lease altered (ADR 0582 clause 4).
on, _ := buscompose.LeaseHandedAway(value, len(value) > 0, nil)
f.mu.Lock()
defer f.mu.Unlock()
was, seen := f.leaseOn, f.leaseSeen
f.leaseOn, f.leaseSeen = on, true
return !seen || was != on || on
}
if state != ReadHave {
return true
}
@@ -186,21 +128,18 @@ func (f *Follower) open() {
}
if f.Hear != nil && !f.heard_ {
err := f.Hear(func(event string, body []byte) {
// An account the vault minted, or a bucket's expiry set (the lease's, altered: ADR 0582 clause 4).
if IsAccountIssued(event) || IsBucketExpirySet(event) {
// An account the vault minted.
if IsAccountIssued(event) {
f.Wake()
return
}
// A condition event of the seat the take-over key does not name is not heard (ADR 0564 §9).
if from := HeardFrom(event); from != "" && from != f.Follows() {
return
}
if IsRootNotFree(body) {
// A condition event is heard from the keeper alone (ADR 0590 §3).
if FromKeeper(event) && IsRootNotFree(body) {
f.Wake()
}
})
if err != nil {
f.logf("the controller's condition events cannot be heard yet (%v): tried again at the next change", err)
f.logf("the keeper's condition events cannot be heard yet (%v): tried again at the next change", err)
} else {
f.heard_ = true
}
modules/mesh-grants/internal/grants/grants.go
+14 -16
View File
@@ -15,9 +15,8 @@ import (
// **The grants the part writes** (novox/hq ADR 0547 §4, §5, ADR 0561 §2, §6; issue 774). A module's grant is the record
// that its bus credential exists for the fingerprint the catalogue's trunk names for it: `<module>.<fingerprint hex>` in
// the part's state grants, naming the accounts (the module's own and each process's) the vault holds a hash for. It is the
// controller's grant as the controller composes it (cmd/mesh-controller grants.go grantsFrom), so the two compare by key.
// Until the controller's lease is handed away nobody reads the part's keys: the assigner reads the controller's
//, and the tool grants says every difference, so the switch is made on a measured equality.
// grant the controller composed (cmd/mesh-controller grants.go grantsFrom), whose readers it took over; the assigner reads
// the part's keys alone since the controller ended by its unassignment (ADR 0590, issue 1126).
//
// A module of which an account has no hash yet is granted the accounts that have one; a module none of whose accounts has
// one has no grant, and one it was written for is deleted (an older grant naming a node whose account is gone would let the
@@ -35,7 +34,7 @@ type GrantsWritten struct {
// `<module>: <why>`: the assigner's sixth rule wants none of it (MissingGrants). Never passed over in silence: once
// the part's grants are the ones read, it is the condition mesh-grants.grant-missing.
Missing []string `json:"missing,omitempty"`
// Waits is each grant composed and not written, `<key>: <why>`: after take-over a grant is new or changed only once the
// Waits is each grant composed and not written, `<key>: <why>`: a grant is new or changed only once the
// list the bus loaded carries it (issue 774, GrantWaits). Read with grants_grants (last_pass).
Waits []string `json:"waits,omitempty"`
}
@@ -46,9 +45,9 @@ type GrantsWritten struct {
// names): every publish and subscribe allowed, the allowance of responses, and no deny the composed account lacks. A grant
// that adds no permission is written at once, whatever list is held or pending; one that adds any waits until a list that
// carries it is loaded. loaded is the loaded list by user, nil when its contents are unknown (unknown says why): then only
// a grant naming no account is written. In shadow nobody reads the part's grants and nothing waits. Pure.
func GrantWaits(tookOver bool, accounts []string, composed, loaded map[string]User, unknown string) string {
if !tookOver || len(accounts) == 0 {
// a grant naming no account is written. Pure.
func GrantWaits(accounts []string, composed, loaded map[string]User, unknown string) string {
if len(accounts) == 0 {
return ""
}
if loaded == nil {
@@ -267,7 +266,7 @@ const GrantMissingCondition = "mesh-grants.grant-missing"
// in the vault (drop). **The fingerprint is the indexer's, never a commit's** (issue 1126: link2pay, snake, f1-circuits
// and txt-game, whose source commit has not moved since a build of the old form, have a done key of the new form at it,
// and that is the fingerprint granted). Pure.
func MissingGrants(in Inputs, want map[string]delivery.ControllerGrant, drop []string, skip map[string]bool) []string {
func MissingGrants(in Inputs, want map[string]delivery.Grant, drop []string, skip map[string]bool) []string {
placed := map[string]bool{}
for _, ms := range in.Assigned {
for _, m := range ms {
@@ -348,7 +347,7 @@ func (p *Part) sayMissing(missing []string) {
// GrantsOf is the grant of every module at the fingerprint the indexer's trunk names for it, by key, and the keys that
// have none (drop): see the comment above. A module in skip (its manifest was refused) is neither. Pure.
func GrantsOf(in Inputs, recs buscompose.Records, skip map[string]bool) (map[string]delivery.ControllerGrant, []string, error) {
func GrantsOf(in Inputs, recs buscompose.Records, skip map[string]bool) (map[string]delivery.Grant, []string, error) {
principals, err := buscompose.Users(recs)
if err != nil {
return nil, nil, err
@@ -364,7 +363,7 @@ func GrantsOf(in Inputs, recs buscompose.Records, skip map[string]bool) (map[str
}
accounts[p.Module] = append(accounts[p.Module], p.Username())
}
out := map[string]delivery.ControllerGrant{}
out := map[string]delivery.Grant{}
var drop []string
for module, fp := range in.Fingerprints {
if skip[module] || !strings.HasPrefix(fp, delivery.SourcePrefix2) {
@@ -380,7 +379,7 @@ func GrantsOf(in Inputs, recs buscompose.Records, skip map[string]bool) (map[str
continue
}
sort.Strings(names)
out[key] = delivery.ControllerGrant{Accounts: names}
out[key] = delivery.Grant{Accounts: names}
}
sort.Strings(drop)
return out, drop, nil
@@ -421,14 +420,13 @@ func (p *Part) writeGrants(res *Result, in Inputs, recs buscompose.Records, comp
done := &GrantsWritten{Wanted: len(want), Missing: MissingGrants(in, want, drop, skip)}
res.Grants = done
p.sayMissing(done.Missing)
tookOver := p.SwitchState() == TookOver
byUser := map[string]User{}
for _, u := range composed {
byUser[u.User] = u
}
for _, key := range sortedKeys(want) {
g := want[key]
data, err := delivery.EncodeControllerGrant(g)
data, err := delivery.EncodeGrant(g)
if err != nil {
res.Errors = append(res.Errors, fmt.Sprintf("the grant %s cannot be encoded: %v", key, err))
continue
@@ -437,11 +435,11 @@ func (p *Part) writeGrants(res *Result, in Inputs, recs buscompose.Records, comp
res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be read: %v", StateGrants, key, err))
continue
} else if found {
if was, err := delivery.DecodeControllerGrant(raw); err == nil && delivery.SameControllerGrant(was, withVersion(g)) {
if was, err := delivery.DecodeGrant(raw); err == nil && delivery.SameGrant(was, withVersion(g)) {
continue
}
}
if waits := GrantWaits(tookOver, g.Accounts, byUser, loaded, unknown); waits != "" {
if waits := GrantWaits(g.Accounts, byUser, loaded, unknown); waits != "" {
done.Waits = append(done.Waits, key+": "+waits)
continue
}
@@ -465,7 +463,7 @@ func (p *Part) writeGrants(res *Result, in Inputs, recs buscompose.Records, comp
}
}
func withVersion(g delivery.ControllerGrant) delivery.ControllerGrant {
func withVersion(g delivery.Grant) delivery.Grant {
g.V = delivery.Version
return g
}
modules/mesh-grants/internal/grants/inputs.go
+12 -13
View File
@@ -1,10 +1,9 @@
// Package grants is the grants part in shadow (novox/hq ADR 0547 clauses 5 to 8, issue 1945): it composes the bus's user
// list, each module's membership and each manifest's buckets, logs and streams from what the bus holds, with the bus
// composer the controller uses (mesh-sdk buscompose), and compares them with what the bus loaded and holds. Beyond its own
// state it takes effect in two things alone, each by a grant the operator allowed: it publishes memberships (2026-10-25
// 22:31), and creates a bucket, log or stream a manifest declares that the bus lacks (2026-10-26 08:54, issue 1790). It
// changes no stream the bus holds, writes no grant and no user list the bus node reads, until the operator's take-over,
// which the operator asks on their channel and the part runs on their yes, after its refusals.
// Package grants is the grants part (novox/hq ADR 0547 clauses 5 to 8, issue 1945): it composes the bus's user list, each
// module's membership and each manifest's buckets, logs and streams from what the bus holds, with the bus composer
// (mesh-sdk buscompose), and compares them with what the bus loaded and holds. It publishes memberships (2026-10-25
// 22:31), creates a bucket, log or stream a manifest declares that the bus lacks (2026-10-26 08:54, issue 1790), writes
// the grants the assigner reads and the user list the bus node follows alone, and makes the account consumers, since the
// controller ended by its unassignment (ADR 0590, issue 1126). It changes no stream the bus holds.
package grants
import (
@@ -46,7 +45,7 @@ const (
// Reads is every state the part reads, in the manifest's order. Not the controller's assignments: the seat's placement
// alone places modules (novox/hq ADR 0549 §5, ADR 0571 §10; issue 774).
var Reads = []string{ReadModules, ReadManifests, ReadResolved, ReadHave, ReadWants, ReadBuilds,
ReadAccounts, ReadPlacement, ReadLease, ReadTakeover, ReadEnrolments, ReadNodeLogins}
ReadAccounts, ReadPlacement, ReadEnrolments, ReadNodeLogins}
// The bus module's tools the part calls, as its manifest's invokes name them: read-only, each.
const (
@@ -56,7 +55,7 @@ const (
)
// Invokes is every tool the part calls.
var Invokes = append([]string{CallUsers, CallUserCan, CallStreams, CallRootFree, CallRecordWarrant}, LeaseCalls...)
var Invokes = []string{CallUsers, CallUserCan, CallStreams, CallRootFree}
// Reader reads another part's state, named `<module>.<state>`. A refusal (a read not granted yet, a bucket not there) is
// an error the part names as a gap and composes past: it never stops for one.
@@ -74,7 +73,7 @@ type Caller interface {
type Gap struct {
Input string `json:"input"`
Why string `json:"why"`
// Failed says the gap is a read that failed this pass, not one that stands (ADR 0547 clause 7: after take-over a
// Failed says the gap is a read that failed this pass, not one that stands (ADR 0547 clause 7: a
// list composed past a failed read is not sealed).
Failed bool `json:"failed,omitempty"`
}
@@ -135,8 +134,8 @@ type Inputs struct {
Gaps []Gap
}
// HeldHash is one key of the vault's bus-accounts: the hash, and what it is of. The hash is never said or written by
// the part in shadow.
// HeldHash is one key of the vault's bus-accounts: the hash, and what it is of. The hash is never said by the part, and
// written only into the list sealed to the bus node.
type HeldHash struct {
Hash string `json:"hash"`
Generation int `json:"generation,omitempty"`
@@ -246,7 +245,7 @@ func ReadInputs(r Reader, cache *Manifests) Inputs {
nodes := map[string]bool{}
// **The seat's placement alone places modules** (novox/hq ADR 0549 §5, ADR 0567 §9; issue 774): the controller's
// assignments are read no more, and the seats held once for the mesh are held where it says. Not read is a failed
// read: nothing is placed, so after take-over the list composed past it is not sealed.
// read: nothing is placed, so the list composed past it is not sealed.
if keys, err := r.Keys(ReadPlacement); err != nil {
in.Gaps = append(in.Gaps, Gap{Input: ReadPlacement, Why: "refused or absent: nothing is placed (" + err.Error() + ")", Failed: true})
} else {
modules/mesh-grants/internal/grants/leasecheck.go
-280
View File
@@ -1,280 +0,0 @@
package grants
import (
"encoding/json"
"fmt"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// **The controller's lease, what still depends on the controller, and a dry run of the lease write's check** (issue 774,
// design 47 L3; novox/hq ADR 0547 §11, ADR 0561 §3, ADR 0571 §13). One read-only answer from bus state: the lease key,
// every manifest the indexer holds, and the bus module's streams. It writes nothing, and asks the controller nothing.
//
// It lives on the grants part because the part is where the lease write takes most of its effect and already reads all
// it needs: the lease (buscompose.NamedReaders names it), every manifest at the indexer's trunk, placement, and the bus's
// streams through the bus module's read tools. No record names the home of the lease write's check; ADR 0571 §13 names
// its test in three repositories, and this is the same judgement over the manifests the mesh holds now.
// ToolLeaseCheck is the tool's name.
const ToolLeaseCheck = "grants_lease_check"
// Controller is the module and the seat the check is about.
const Controller = "mesh-controller"
// **The judge is mesh-sdk's** (catalogue.Dependencies, catalogue.Judge; issue 774): the same judgement the three
// repositories' manifest tests run through catalogue.ManifestRefusals (ADR 0571 §13), re-exported here by its names.
const (
DependsRead = catalogue.DependsRead
DependsInvoke = catalogue.DependsInvoke
DependsConsume = catalogue.DependsConsume
DependsUses = catalogue.DependsUses
DependsWaits = catalogue.DependsWaits
PhaseBefore = catalogue.PhaseBefore
PhaseWrite = catalogue.PhaseWrite
)
type (
// Dependency, Kept, Verdict and Coverage are mesh-sdk's.
Dependency = catalogue.Dependency
Kept = catalogue.KeptDependency
Verdict = catalogue.Verdict
Coverage = catalogue.Coverage
)
// KeepList and EndedBy are mesh-sdk's named list and the records that ended what it no longer keeps.
var (
KeepList = catalogue.KeepList
EndedBy = catalogue.EndedBy
)
// Dependencies are every declaration on the controller in the manifests given (catalogue.Dependencies).
func Dependencies(manifests map[string]catalogue.Manifest) []Dependency {
return catalogue.Dependencies(manifests)
}
// Judge runs the lease write's check in one phase (catalogue.Judge).
func Judge(deps []Dependency, phase string, cov Coverage, handedAway bool, leaseSays string) Verdict {
return catalogue.Judge(deps, phase, cov, handedAway, leaseSays)
}
// LeaseNow is who holds the lease and what follows from it.
type LeaseNow struct {
HandedAway bool `json:"handed_away"`
Says string `json:"says"`
Holder *buscompose.LeaseHolder `json:"holder,omitempty"`
Since *time.Time `json:"since,omitempty"`
Does []string `json:"does"`
}
// While the controller holds the lease, and once it is handed away: what the holder and the parts watching the key do
// because of it, by the records that say so.
var (
DoesWhileController = []string{
"the controller mints every new bus account; the vault mints none and says what it holds back (ADR 0561 §3)",
"the grants part creates no durable consumer an account needs, and only compares them (ADR 0561 §6)",
"the grants part publishes memberships on the controller's assignments stream beside its own state (ADR 0571 §10)",
"mesh-healers writes each act it would take in `would`, and acts on none (ADR 0564 §8)",
"the artifact sweep lets go of nothing; the controller's take-in sweeps and says `artifacts-gone` (ADR 0571 §4)",
"the watchman only lists dead letters; the controller keeps DEAD_LETTERS through its consumer of DEAD_LETTER_NOTICES (ADR 0564 §5)",
"the controller keeps composing for the bus node, whose agent follows its user list, so the hand-backs of config, conditions, placement and the user list still answer (ADR 0547 §8, ADR 0564 §9, ADR 0565 §4, ADR 0567 §11)",
}
DoesHandedAway = []string{
"the vault mints every new bus account (ADR 0561 §1)",
"the grants part creates the durable consumers an account needs, and stops publishing on the assignments stream (ADR 0561 §6, ADR 0571 §10)",
"mesh-healers acts (ADR 0564 §8)",
"the artifact sweep sweeps and records every deletion in the hand-act log (ADR 0571 §4)",
"the watchman keeps the dead letters (ADR 0564 §5)",
"every hand-back is answered as gone (ADR 0547 §11, ADR 0564 §9, ADR 0565 §4, ADR 0567 §11)",
}
)
// Lease reads the lease key and says who holds it, since when, and what follows.
func Lease(r Reader) LeaseNow {
raw, found, err := r.Get(ReadLease, LeaseKey)
away, says := buscompose.LeaseHandedAway(raw, found, err)
out := LeaseNow{HandedAway: away, Says: says, Does: DoesWhileController}
if away {
out.Does = DoesHandedAway
}
if err == nil && found && len(raw) > 0 {
var h buscompose.LeaseHolder
if json.Unmarshal(raw, &h) == nil {
out.Holder = &h
if !h.Taken.IsZero() {
t := h.Taken
out.Since = &t
}
}
}
return out
}
// StreamUse is one of the controller's streams (its buckets, the assignments stream, the dead-letter notices) as the bus
// module answers it: who consumes it now, and what it last carried.
type StreamUse struct {
Stream string `json:"stream"`
Consumers int `json:"consumers"`
Durable []string `json:"durable,omitempty"`
Messages int64 `json:"messages"`
Subjects int64 `json:"subjects_held,omitempty"`
Last string `json:"last,omitempty"`
}
// The streams of the controller's that are not its buckets: the assignments stream it publishes memberships on, and the
// notices its consumer `controller` turns into dead letters until the lease write (ADR 0564 §5).
const (
StreamAssignments = "ASSIGNMENTS"
StreamDeadLetterNotices = "DEAD_LETTER_NOTICES"
)
// controllerStream says whether a stream is the controller's.
func controllerStream(name string) bool {
return strings.HasPrefix(name, "KV_"+Controller+"_") || name == StreamAssignments || name == StreamDeadLetterNotices
}
// Duties reads, from the bus module's streams, every stream of the controller's someone consumes, with its durable
// consumers by name (a watcher of a bucket is ephemeral and names nobody). Streams with no consumer are counted apart.
func Duties(c Caller) (used []StreamUse, idle []string, err error) {
raw, err := c.Call(CallStreams, map[string]any{})
if err != nil {
return nil, nil, fmt.Errorf("%s: %w", CallStreams, err)
}
type consumer struct {
Name string `json:"name"`
Kind string `json:"kind"`
Max int `json:"max_deliver"`
Ack string `json:"ack_policy"`
}
type stream struct {
Name string `json:"name"`
Consumers int `json:"consumers"`
Messages int64 `json:"messages"`
Held int64 `json:"subjects_held"`
Last string `json:"last"`
Detail []consumer `json:"consumer_detail"`
}
var got struct {
Detail []stream `json:"detail"`
}
if err := json.Unmarshal(unwrap(raw), &got); err != nil {
return nil, nil, fmt.Errorf("%s answered no streams: %w", CallStreams, err)
}
for _, s := range got.Detail {
if !controllerStream(s.Name) {
continue
}
if s.Consumers == 0 {
idle = append(idle, s.Name)
continue
}
u := StreamUse{Stream: s.Name, Consumers: s.Consumers, Messages: s.Messages, Subjects: s.Held, Last: s.Last}
if one, err := c.Call(CallStreams, map[string]any{"stream": s.Name}); err == nil {
var d struct {
Detail []stream `json:"detail"`
}
if json.Unmarshal(unwrap(one), &d) == nil && len(d.Detail) > 0 {
for _, cs := range d.Detail[0].Detail {
if durable(cs.Name) {
u.Durable = append(u.Durable, cs.Name)
}
}
}
}
used = append(used, u)
}
sort.Slice(used, func(i, j int) bool { return used[i].Stream < used[j].Stream })
sort.Strings(idle)
return used, idle, nil
}
// durable says whether a consumer's name is one a part gave it: the bus names an ephemeral watcher with eight random
// letters and digits, which no part of the mesh uses.
func durable(name string) bool {
if len(name) != 8 {
return true
}
for _, r := range name {
if !(r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9') {
return true
}
}
return false
}
// LeaseAnswer is the tool's answer.
type LeaseAnswer struct {
Lease LeaseNow `json:"lease"`
Dependencies []Dependency `json:"dependencies"`
// Unkept are the dependencies no record keeps, by module: what the check refuses in every phase.
Unkept map[string][]string `json:"unkept"`
Manifests int `json:"manifests_read"`
// Indexed is how many modules the indexer holds; -1 when its keys could not be listed.
Indexed int `json:"modules_indexed"`
Unread []Gap `json:"manifests_unread,omitempty"`
// Streams are the controller's streams someone consumes now; Idle those no one does.
Streams []StreamUse `json:"streams,omitempty"`
Idle []string `json:"streams_idle,omitempty"`
StreamsError string `json:"streams_unread,omitempty"`
// Check is the dry run at the write (the three alone), Before the same over the whole list before the write's
// delivery.
Check Verdict `json:"check"`
Before Verdict `json:"check_before_delivery"`
Beside []string `json:"not_judged_here"`
}
// NotJudgedHere are the lease write's other conditions, each with the tool that answers it (design 47 L3, ADR 0547 §11).
var NotJudgedHere = []string{
"L4 to L7 delivered: the issue tracker's milestones 1952, 1953, 1954 and 1955",
"every node's record trusts the vault's key: mesh-delivery-view.node_trusts (ADR 0550 §2)",
"mesh-config's compare empty and its take-over done, and mesh-wants.upgrade compare empty: mesh-config.compare, mesh-wants.upgrade (ADR 0565 §14)",
"the condition keeper's take-over done: mesh-conditions.compare (ADR 0564 §9)",
}
// LeaseCheck answers the tool: who holds the lease and what follows, every dependency on the controller with what keeps
// it, the controller's streams in use, and the lease write's check run dry. Read-only.
func (p *Part) LeaseCheck() (any, error) {
in := ReadInputs(p.Read, &p.cache)
out := LeaseAnswer{Lease: Lease(p.Read), Unkept: map[string][]string{}, Manifests: len(in.Manifests), Unread: in.Refused,
Beside: NotJudgedHere}
for _, g := range in.Gaps {
if g.Failed && strings.HasPrefix(g.Input, ReadModules) {
out.Unread = append(out.Unread, g)
}
}
out.Dependencies = Dependencies(in.Manifests)
if out.Dependencies == nil {
out.Dependencies = []Dependency{}
}
for _, d := range out.Dependencies {
if d.Kept == "" {
out.Unkept[d.Module] = append(out.Unkept[d.Module], d.Kind+" "+d.What)
}
}
if used, idle, err := Duties(p.Call); err != nil {
out.StreamsError = err.Error()
} else {
out.Streams, out.Idle = used, idle
}
// Only the indexer's own modules count as read: a manifest the mesh provides itself must not stand in for one missing.
cov := Coverage{Indexed: -1}
for _, g := range out.Unread {
cov.Unread = append(cov.Unread, catalogue.Unread{Input: g.Input, Why: g.Why})
}
if keys, err := p.Read.Keys(ReadModules); err == nil {
cov.Indexed = len(keys)
for _, k := range keys {
if _, ok := in.Manifests[k]; ok {
cov.Read++
}
}
}
out.Indexed = cov.Indexed
out.Check = Judge(out.Dependencies, PhaseWrite, cov, out.Lease.HandedAway, out.Lease.Says)
out.Before = Judge(out.Dependencies, PhaseBefore, cov, out.Lease.HandedAway, out.Lease.Says)
return out, nil
}
modules/mesh-grants/internal/grants/leasewrite.go
-962
View File
@@ -1,962 +0,0 @@
package grants
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/handacts"
"git.novox.be/novox/mesh-sdk/go/health"
)
// **The lease write, a verb of the seat mesh-grants** (novox/hq ADR 0582, issue 774). `lease-write` takes a why and is
// refused, naming the check, until every precondition of clause 2 reads green; otherwise the operator is asked once, on
// a channel that proves who answered, with the checks shown and their digest bound to the yes (clause 1). On the
// warrant the checks are read again, and then, each step read back before the next (clause 3): the controller's service
// is stopped on its node and read back stopped; the bus's user list is composed without the controller's write on the
// lease (buscompose.Records.LeaseCut) and waited on until the bus node's have says it loaded it; the lease bucket's
// expiry is removed through the bus module's tool; the key `holder` names mesh-grants with the operator, the why, the
// ask and the time; `lease-handed` is said, the act recorded through mesh-hand-acts.warranted, and `removal-due` is
// written in the state `lease` with a day to live. A lease altered after it is said as `lease-altered`, once, and
// nothing is written again until the operator asks (clause 4). `lease-hand-back` reverses it for a day (clause 5). The
// expiry marker of `removal-due`, and nothing else, starts the day-later removal of the controller's consumer of
// DEAD_LETTER_NOTICES and the stream ASSIGNMENTS (clause 6). Every step waits on an event (a warrant, the bus node's
// have, a marker), never on a timer.
// The verbs, events and states of the lease write.
const (
ActLeaseWrite = catalogue.GrantsVerbLeaseWrite
ActLeaseHandBack = catalogue.GrantsVerbLeaseHandBack
EventLeaseHanded = catalogue.GrantsEventLeaseHanded
// EventLeaseAltered and EventRemovalFailed are the seat's other two events.
EventLeaseAltered = catalogue.GrantsEventLeaseAltered
EventRemovalFailed = catalogue.GrantsEventRemovalFailed
// StateLease is the part's state with a time to live per key, holding KeyRemovalDue alone.
StateLease = "lease"
KeyRemovalDue = "removal-due"
// KeyLease is the lease write's record in StateTakeover: the cut, the write, the removal and every act.
KeyLease = "lease"
// ToolRemovalDue is the tool that writes removal-due again with a short time to live, the recovery for a marker
// missed while the part was down.
ToolRemovalDue = "grants_removal_due"
)
// RemovalAfter is how long after the write the controller's consumer and stream are removed (ADR 0582 clause 6).
const RemovalAfter = 24 * time.Hour
// The calls the lease write makes, as the manifest's invokes name them.
const (
// CallServiceStop, CallServiceStart and CallServiceStatus are the node seat's verbs, asked of the controller's node.
CallServiceStop = "seat:" + catalogue.ServiceManagerSeat + ".stop"
CallServiceStart = "seat:" + catalogue.ServiceManagerSeat + ".start"
CallServiceStatus = "seat:" + catalogue.ServiceManagerSeat + ".status"
// CallBucketExpiry sets and restores a bucket's expiry, recording the act itself; CallConsumerRemove and
// CallStreamRemove are the removal's two tools.
CallBucketExpiry = "nats.nats_bucket_expiry"
CallConsumerRemove = "nats.nats_consumer_remove"
CallStreamRemove = "nats.nats_stream_remove"
// CallRecordHandAct records the removals as hand acts (ADR 0582 clause 6, ADR 0565 §11).
CallRecordHandAct = "seat:" + handacts.Seat + ".hand-act"
// The checks' calls (clause 2).
CallNodeTrusts = "mesh-delivery-view.node_trusts"
CallConfigCompare = "seat:mesh-config.compare"
CallUpgrade = "seat:" + catalogue.WantsSeat + ".upgrade"
CallVaultStatus = "mesh-vault.vault_status"
CallIssue = "seat:issue-tracker.get"
)
// LeaseCalls are every call the lease write makes.
var LeaseCalls = []string{CallServiceStop, CallServiceStart, CallServiceStatus, CallBucketExpiry, CallConsumerRemove,
CallStreamRemove, CallRecordHandAct, CallNodeTrusts, CallConfigCompare, CallUpgrade, CallVaultStatus, CallIssue}
// ControllerUnit is the controller's service on its node.
const ControllerUnit = Controller + ".service"
// LeaseExpiry is the lease bucket's expiry a hand-back restores (buscompose.LeaseTTL).
var LeaseExpiry = buscompose.LeaseTTL
// RemovedTools are the bus module's tools the lease write and its removal need: the check reads them in its manifest.
var RemovedTools = []string{"nats_bucket_expiry", "nats_consumer_remove", "nats_stream_remove"}
// EarlierSteps are the issues of design 47's earlier steps the write waits on (ADR 0582 clause 2: issues 1952-1955 delivered).
var EarlierSteps = []uint64{1952, 1953, 1954, 1955}
// LeaseBus is the process's hand on the controller's lease: the key `holder` put and deleted, and the bucket's expiry
// read. Nil in the tools, which ask the process.
type LeaseBus interface {
PutHolder(value []byte) error
DeleteHolder() error
Expiry() (time.Duration, error)
}
// TTLStore is a Store that writes a key with its own time to live (the state lease).
type TTLStore interface {
PutTTL(state, key string, value []byte, ttl time.Duration) (uint64, error)
}
// SeatSayer says one of the seat's events, `mesh.seat.mesh-grants.event.<event>`.
type SeatSayer func(event string, body any) error
// Check is one precondition as read: its name, whether it reads green, and what it says.
type Check struct {
Name string `json:"name"`
Green bool `json:"green"`
Says string `json:"says"`
}
// checksDigest is the digest of the checks as shown: what the yes binds.
func checksDigest(cs []Check) string {
raw, _ := json.Marshal(cs)
sum := sha256.Sum256(raw)
return "sha256:" + hex.EncodeToString(sum[:])
}
// LeaseAct is one lease-write or lease-hand-back: asked, answered and each step it reached.
type LeaseAct struct {
Act string `json:"act"`
By string `json:"by"`
Why string `json:"why"`
At time.Time `json:"at"`
Ask string `json:"ask"`
// Checks are the preconditions as the ask showed them, and Digest their digest the yes binds.
Checks []Check `json:"checks,omitempty"`
Digest string `json:"digest,omitempty"`
Raised bool `json:"raised,omitempty"`
// Step is where it stands: asked; then on the warrant stopped, cut, loaded, expiry, written, done for a write and
// restored, uncut, done for a hand-back; or declined, expired, refused, failed.
Step string `json:"step"`
Said string `json:"said,omitempty"`
// List is the digest of the list composed at the cut (or without it, for a hand-back), awaited loaded.
List string `json:"list,omitempty"`
Done time.Time `json:"done,omitzero"`
}
// The steps an act reaches.
const (
StepAsked = "asked"
StepStopped = "stopped"
StepCut = "cut"
StepWritten = "written"
StepRestored = "restored"
StepUncut = "uncut"
StepDone = "done"
StepDeclined = "declined"
StepExpired = "expired"
StepRefused = "refused"
StepFailed = "failed"
)
// LeaseRecord is the key `lease` of StateTakeover.
type LeaseRecord struct {
V int `json:"v"`
// Cut composes the list with buscompose.Records.LeaseCut: from the write's second step until a hand-back's.
Cut bool `json:"cut"`
// Node is the controller's node, from the lease's holder at the write.
Node string `json:"node,omitempty"`
// Handed is when the key holder was written, zero when not (or handed back); Ask the write's ask.
Handed time.Time `json:"handed,omitzero"`
Ask string `json:"ask,omitempty"`
// Removed is when the day-later removal was done; zero until then.
Removed time.Time `json:"removed,omitzero"`
// Altered is what lease-altered said, empty while the lease reads as written.
Altered string `json:"altered,omitempty"`
// Open is the act asked or under way.
Open *LeaseAct `json:"open,omitempty"`
// Acts are the acts ended, newest last.
Acts []LeaseAct `json:"acts"`
}
func (p *Part) readLease() (LeaseRecord, error) {
r := LeaseRecord{V: 1, Acts: []LeaseAct{}}
raw, _, found, err := p.Store.Get(StateTakeover, KeyLease)
if err != nil {
return r, fmt.Errorf("the lease write's record cannot be read: %w", err)
}
if found {
if err := json.Unmarshal(raw, &r); err != nil {
return r, fmt.Errorf("the lease write's record does not read: %w", err)
}
}
return r, nil
}
func (p *Part) writeLease(r LeaseRecord) error {
if len(r.Acts) > keptActs {
r.Acts = r.Acts[len(r.Acts)-keptActs:]
}
if r.Acts == nil {
r.Acts = []LeaseAct{}
}
data, _ := json.Marshal(r)
if _, err := p.Store.Put(StateTakeover, KeyLease, data); err != nil {
return fmt.Errorf("the lease write's record cannot be written: %w", err)
}
return nil
}
// leaseCut says whether the list is composed with the cut.
func (p *Part) leaseCut() bool {
r, err := p.readLease()
return err == nil && r.Cut
}
// leaseHanded says whether the key holder names mesh-grants by the part's own write.
func (p *Part) leaseHanded() bool {
r, err := p.readLease()
return err == nil && !r.Handed.IsZero()
}
// end closes the open act with its step and why.
func (r *LeaseRecord) end(step, said string, at time.Time) {
if r.Open == nil {
return
}
a := *r.Open
a.Step, a.Said, a.Done = step, said, at
r.Acts = append(r.Acts, a)
r.Open = nil
}
// LeaseCheckNames are the checks LeaseChecks reads, in its order.
var LeaseCheckNames = []string{"lease-check", "earlier-steps", "node-trusts", "config-compare", "upgrade-compare",
"keeper-take-over", "vault-minting", "bus-tools"}
// LeaseChecks reads every precondition of the write (ADR 0582 clause 2), each named, in one order.
func (p *Part) LeaseChecks() []Check {
var out []Check
add := func(name string, green bool, says string) {
out = append(out, Check{Name: name, Green: green, Says: says})
}
// The lease check at the write, with the three kept to it alone (ADR 0571 §13).
if raw, err := p.LeaseCheck(); err != nil {
add("lease-check", false, "the lease check could not run: "+err.Error())
} else if a, ok := raw.(LeaseAnswer); !ok {
add("lease-check", false, "the lease check answered nothing it reads")
} else {
add("lease-check", a.Check.Passes, strings.Join(a.Check.Reasons, "; "))
}
// Design 47's earlier steps delivered.
var open []string
for _, n := range EarlierSteps {
raw, err := p.Call.Call(CallIssue, map[string]any{"number": n, "brief": true})
var is struct {
Status string `json:"status"`
}
switch {
case err != nil:
open = append(open, fmt.Sprintf("issue %d cannot be read (%v)", n, err))
case json.Unmarshal(unwrap(raw), &is) != nil || is.Status != "resolved":
open = append(open, fmt.Sprintf("issue %d is %q, not resolved", n, is.Status))
}
}
add("earlier-steps", len(open) == 0, said(open, "issues 1952 to 1955 are resolved"))
// Every node trusts the vault's key.
if raw, err := p.Call.Call(CallNodeTrusts, map[string]any{}); err != nil {
add("node-trusts", false, "node_trusts cannot be read: "+err.Error())
} else {
var t struct {
Every bool `json:"every"`
NotYet []string `json:"not_yet"`
}
_ = json.Unmarshal(unwrap(raw), &t)
add("node-trusts", t.Every, said(prefixed("not yet: ", t.NotYet), "every node's record trusts the vault's key"))
}
// mesh-config's compare empty and its take-over done.
if raw, err := p.Call.Call(CallConfigCompare, map[string]any{}); err != nil {
add("config-compare", false, "mesh-config's compare cannot be read: "+err.Error())
} else {
var c struct {
Source string `json:"source"`
Count int `json:"count"`
}
_ = json.Unmarshal(unwrap(raw), &c)
add("config-compare", c.Count == 0 && c.Source == "indexer",
fmt.Sprintf("mesh-config resolves from %q with %d differences", c.Source, c.Count))
}
// The upgrade compare empty.
if raw, err := p.Call.Call(CallUpgrade, map[string]any{"compare": "true"}); err != nil {
add("upgrade-compare", false, "mesh-wants' upgrade compare cannot be read: "+err.Error())
} else {
var u struct {
Empty bool `json:"empty"`
Splits []json.RawMessage `json:"splits"`
}
_ = json.Unmarshal(unwrap(raw), &u)
add("upgrade-compare", u.Empty, fmt.Sprintf("%d policy splits", len(u.Splits)))
}
// The condition keeper's take-over done.
raw, found, err := p.Read.Get(ReadTakeover, health.TakeoverKey)
t, terr := health.DecodeTakeover(raw, found)
switch {
case err != nil:
add("keeper-take-over", false, "the keeper's take-over cannot be read: "+err.Error())
case terr != nil:
add("keeper-take-over", false, "the keeper's take-over does not read: "+terr.Error())
default:
add("keeper-take-over", t.Follow != health.FollowController, "conditions follow "+t.Follow)
}
// The vault minting with nothing held back.
if raw, err := p.Call.Call(CallVaultStatus, map[string]any{}); err != nil {
add("vault-minting", false, "vault_status cannot be read: "+err.Error())
} else {
var v struct {
Ready bool `json:"ready"`
Minting struct {
On bool `json:"on"`
HeldBack map[string]string `json:"held_back"`
} `json:"minting"`
}
_ = json.Unmarshal(unwrap(raw), &v)
add("vault-minting", v.Ready && v.Minting.On && len(v.Minting.HeldBack) == 0,
fmt.Sprintf("ready %v, minting %v, %d held back", v.Ready, v.Minting.On, len(v.Minting.HeldBack)))
}
// The bus module's tools for the expiry and the removal live: its manifest at the indexer's trunk lists them.
in := ReadInputs(p.Read, &p.cache)
var missing []string
for _, tool := range RemovedTools {
if !slices.Contains(in.Manifests[BusModule].Tools, tool) {
missing = append(missing, tool)
}
}
add("bus-tools", len(missing) == 0, said(prefixed("the bus module lists no ", missing), "the bus module lists its three tools"))
return out
}
func said(problems []string, ok string) string {
if len(problems) == 0 {
return ok
}
return strings.Join(problems, "; ")
}
func prefixed(prefix string, items []string) []string {
if len(items) == 0 {
return nil
}
return []string{prefix + strings.Join(items, ", ")}
}
// notGreen is every check not green, `<name>: <says>`.
func notGreen(cs []Check) []string {
var out []string
for _, c := range cs {
if !c.Green {
out = append(out, c.Name+": "+c.Says)
}
}
return out
}
// changedChecks names each check whose reading differs from the one shown.
func changedChecks(shown, now []Check) []string {
was := map[string]Check{}
for _, c := range shown {
was[c.Name] = c
}
var out []string
for _, c := range now {
if w, ok := was[c.Name]; !ok || w != c {
out = append(out, c.Name+" (now: "+c.Says+")")
}
delete(was, c.Name)
}
for name := range was {
out = append(out, name+" (no longer read)")
}
slices.Sort(out)
return out
}
// LeaseWrite is the verb lease-write: refused, naming each check not green, or asked of the operator by the part's process.
func (p *Part) LeaseWrite(caller, why string) (any, error) {
return p.leaseRequest(ActLeaseWrite, caller, why)
}
// LeaseHandBack is the verb lease-hand-back.
func (p *Part) LeaseHandBack(caller, why string) (any, error) {
return p.leaseRequest(ActLeaseHandBack, caller, why)
}
func (p *Part) leaseRequest(act, caller, why string) (any, error) {
if strings.TrimSpace(why) == "" {
return nil, fmt.Errorf("%s needs a why, shown to the operator and kept", act)
}
r, err := p.readLease()
if err != nil {
return nil, err
}
if r.Open != nil {
return nil, fmt.Errorf("%s is already under way: the ask %s stands at %s", r.Open.Act, r.Open.Ask, r.Open.Step)
}
var checks []Check
switch act {
case ActLeaseWrite:
if !r.Handed.IsZero() && r.Altered == "" {
return nil, errors.New("the lease is handed to mesh-grants already: nothing to write")
}
checks = p.LeaseChecks()
if bad := notGreen(checks); len(bad) > 0 {
return nil, fmt.Errorf("lease-write is refused until every check reads green (ADR 0582 clause 2): %s", strings.Join(bad, "; "))
}
case ActLeaseHandBack:
if refused := handBackRefused(r); refused != "" {
return nil, errors.New(refused)
}
}
a := LeaseAct{Act: act, By: caller, Why: why, At: p.now(), Checks: checks, Digest: checksDigest(checks), Step: StepAsked}
a.Ask = act + "-" + p.now().Format("20060102t150405") + "-" + p.nonce()
r.Open = &a
if err := p.writeLease(r); err != nil {
return nil, err
}
_, _ = p.AskCompose(caller)
return map[string]any{"asked": a.Ask, "checks": checks, "says": "the part's process asks the operator on their channel; " +
"only the operator's yes runs it, after the checks are read again. grants_lease_check shows the lease, and " +
"the record is the key lease of the state takeover"}, nil
}
// handBackRefused is why a hand-back cannot be asked, or empty.
func handBackRefused(r LeaseRecord) string {
switch {
case !r.Removed.IsZero():
return "lease-hand-back is refused: the day-later removal ran, so the controller would start without its " +
"consumer of DEAD_LETTER_NOTICES and the stream ASSIGNMENTS (ADR 0582 clause 5)"
case r.Handed.IsZero() && !r.Cut:
return "lease-hand-back is refused: the lease is not handed to mesh-grants, so there is nothing to hand back"
}
return ""
}
// EventBucketExpirySet is the bus module's event of a bucket's expiry set, which the part consumes: a lease bucket's
// expiry changed is heard, never polled.
const EventBucketExpirySet = BusModule + ".bucket-expiry-set"
// IsBucketExpirySet says an event heard is the bus module's bucket-expiry-set.
func IsBucketExpirySet(event string) bool {
return event == EventBucketExpirySet || event == "mesh.mod."+BusModule+".event.bucket-expiry-set"
}
// isLeaseAsk says an ask is the lease write's.
func isLeaseAsk(id string) bool {
return strings.HasPrefix(id, ActLeaseWrite+"-") || strings.HasPrefix(id, ActLeaseHandBack+"-")
}
// leaseAskOf is the ask of an act, in the words the operator reads: what it does, the checks it showed, and the one
// option whose yes binds the act, the holder, the ask and the checks' digest (ADR 0582 clause 1).
func (p *Part) leaseAskOf(a LeaseAct) (asks.Ask, error) {
act := asks.Act{"verb": a.Act, "module": Module, "ask": a.Ask, "holder": Module, "checks": a.Digest}
bind, err := asks.ActDigest(act)
if err != nil {
return asks.Ask{}, err
}
var lines []string
for _, c := range a.Checks {
lines = append(lines, "✓ "+c.Name+": "+c.Says)
}
headline, does, stays := "Hand the controller's lease to mesh-grants?",
"The controller is stopped and the lease is handed to mesh-grants for good; hand-back stays open for a day.",
"The controller keeps its lease."
expl := "Every check of the lease write reads green. Your yes stops the controller, takes from it the right to write " +
"its lease, removes the lease's expiry and names mesh-grants as its holder. A controller started later fails on the lease."
if a.Act == ActLeaseHandBack {
headline, does, stays = "Give the controller its lease back?",
"The lease's expiry is restored, mesh-grants lets go of it, the controller gets its rights back and is started.",
"The lease stays with mesh-grants."
expl = "The lease was handed to mesh-grants less than a day ago. Your yes gives it back and starts the controller."
}
return asks.Ask{ID: a.Ask, Headline: headline, Explanation: expl, Who: asks.Operator,
Options: []asks.Option{{ID: "yes", Label: "Yes", Does: does, Level: asks.Approve, Binds: bind},
{ID: "no", Label: "No, leave it", Does: stays, Level: asks.Acknowledge}},
Expires: a.At.Add(asks.ApproveLasts - time.Minute), OnExpiry: stays + " Nothing changes.", About: "mesh-grants-" + a.Act,
Details: "Asked by " + a.By + ": " + a.Why + "\n" + strings.Join(lines, "\n")}, nil
}
// raiseLease asks the operator the open act once; closes one past its expiry; catches up a warrant the process missed.
func (p *Part) raiseLease() error {
if p.Asker == nil {
return nil
}
r, err := p.readLease()
if err != nil || r.Open == nil || r.Open.Step != StepAsked {
return err
}
a := r.Open
if !a.Raised {
ask, err := p.leaseAskOf(*a)
if err != nil {
return err
}
if err := p.Asker.Ask(ask); err != nil {
if p.Forget != nil {
_ = p.Forget(a.Ask)
}
_ = p.Asker.Cancel(a.Ask)
r.end(StepRefused, "the operator could not be asked: "+err.Error(), p.now())
_ = p.writeLease(r)
return fmt.Errorf("the operator could not be asked the %s: %v", a.Act, err)
}
a.Raised = true
return p.writeLease(r)
}
_, w, err := p.Asker.Where(a.Ask)
if err != nil {
return fmt.Errorf("the router's record of the ask %s cannot be read: %v", a.Ask, err)
}
if w != nil {
h, herr := p.Asker.Take(*w, p.Warranted)
if h == "" && herr != nil {
return fmt.Errorf("the operator's answer to %s could not be taken yet: %v", a.Ask, herr)
}
p.Said(h, *w, herr)
return nil
}
if p.now().After(a.At.Add(asks.ApproveLasts + askMargin)) {
r.end(StepExpired, "the ask expired: nothing was written", p.now())
return p.writeLease(r)
}
return nil
}
// leaseWarranted is the operator's answer to a lease act: a no or a refusal writes nothing; a yes runs the act.
func (p *Part) leaseWarranted(o asks.Option, w asks.Warrant) error {
r, err := p.readLease()
if err != nil {
return err
}
if r.Open == nil || r.Open.Ask != w.Ask || r.Open.Step != StepAsked {
return fmt.Errorf("the warrant answers the ask %s, which is not the lease act asked: nothing is written", w.Ask)
}
if o.ID != "yes" {
r.end(StepDeclined, "the operator said no: nothing was written", p.now())
return p.writeLease(r)
}
a := r.Open
if a.Act == ActLeaseWrite {
// Read again on the warrant (ADR 0582 clauses 1, 2): the yes binds the checks shown, so any check that reads
// otherwise now (one no longer green among them) refuses it, named.
if now := p.LeaseChecks(); checksDigest(now) != a.Digest {
r.end(StepRefused, "refused on the warrant, the checks changed since the ask: "+strings.Join(changedChecks(a.Checks, now), "; "), p.now())
p.sayLeaseRefused(a.Act, r.Acts[len(r.Acts)-1].Said)
return p.writeLease(r)
}
} else if refused := handBackRefused(r); refused != "" {
r.end(StepRefused, refused, p.now())
return p.writeLease(r)
}
a.Step = "warranted"
if err := p.writeLease(r); err != nil {
return err
}
return p.advanceLease()
}
// sayLeaseRefused tells the operator a yes was refused, as the take-over's refusal is said.
func (p *Part) sayLeaseRefused(act, why string) {
if p.Say == nil {
return
}
_ = p.Say(asks.Condition{Key: "mesh-grants." + act + "-refused", Severity: asks.SeverityWarning,
Subject: asks.Scope{Scope: "module", ID: Module}, State: asks.StateRaised, Summary: act + " refused: " + why,
Headline: "The lease write was refused", Needs: "a look at why",
Explanation: asks.NeedsYou + "a look at why. You said yes and it was refused: " + why, Resolved: "the act is asked again and goes through"})
}
// advanceLease takes the open act as far as it can now: each step read back before the next, and a step that waits (the
// list loaded) left for the pass that hears the bus node's have. A step that fails ends the act there, saying why, and
// writes nothing more.
func (p *Part) advanceLease() error {
r, err := p.readLease()
if err != nil || r.Open == nil || r.Open.Step == StepAsked {
return err
}
a := r.Open
fail := func(why string) error {
if p.Logf != nil {
p.Logf("%s failed at its step %s: %s", a.Act, a.Step, why)
}
r.end(StepFailed, why, p.now())
p.sayLeaseRefused(a.Act, why)
return p.writeLease(r)
}
if a.Act == ActLeaseHandBack {
return p.advanceHandBack(&r, fail)
}
switch a.Step {
case "warranted":
// 1. Stop the controller's service on its node, read back stopped.
node, why := p.controllerNode()
if node == "" {
return fail(why)
}
if stopped, why := p.service(CallServiceStop, node); !stopped {
return fail("the controller's service on " + node + " did not read back stopped (" + why + "): nothing more is written")
}
r.Node, a.Step = node, StepStopped
fallthrough
case StepStopped:
// 2. Compose the list without the controller's write on the lease.
r.Cut, a.Step = true, StepCut
if err := p.writeLease(r); err != nil {
return err
}
res := p.composeQuiet()
g, _, _ := p.Sealed()
if res.Sealed == nil || g.Digest == "" {
return fail("the list composed with the cut could not be written sealed: " + strings.Join(res.Errors, "; "))
}
if r, err = p.readLease(); err != nil || r.Open == nil {
return err
}
a = r.Open
a.List = g.Digest
if err := p.writeLease(r); err != nil {
return err
}
fallthrough
case StepCut:
// 3. Wait until the bus node's have says it loaded that list.
if !p.loaded(a.List) {
return nil
}
// 4. Remove the lease bucket's expiry.
if ok, why := p.setExpiry("none", *a); !ok {
return fail("the lease's expiry was not removed (" + why + "): the holder is not written")
}
// 5. Create the key holder naming mesh-grants.
h := buscompose.LeaseHolder{HandedTo: Module, HandedBy: "operator", Why: a.Why, Ask: a.Ask, At: p.now(), Host: r.Node}
raw, _ := json.Marshal(h)
if p.Lease == nil {
return fail("the part holds no hand on the lease here")
}
if err := p.Lease.PutHolder(raw); err != nil {
return fail("the key holder could not be written: " + err.Error())
}
r.Handed, r.Ask, r.Altered, a.Step = p.now(), a.Ask, "", StepWritten
if err := p.writeLease(r); err != nil {
return err
}
// 6. Say lease-handed, record the act, and write removal-due with a day to live.
p.saySeat(EventLeaseHanded, map[string]any{"holder": Module, "ask": a.Ask, "why": a.Why, "node": r.Node, "at": r.Handed})
p.recordWarrant(a.Ask)
if err := p.putRemovalDue(RemovalAfter); err != nil {
r.end(StepDone, "the lease is handed; removal-due could not be written ("+err.Error()+"): grants_removal_due writes it", p.now())
} else {
r.end(StepDone, "the lease is handed to mesh-grants; the removal is due in a day", p.now())
}
return p.writeLease(r)
}
return nil
}
// advanceHandBack restores the expiry, deletes holder and removal-due, composes without the cut, and once that list is
// loaded starts the controller (ADR 0582 clause 5).
func (p *Part) advanceHandBack(r *LeaseRecord, fail func(string) error) error {
a := r.Open
switch a.Step {
case "warranted":
if ok, why := p.setExpiry(LeaseExpiry.String(), *a); !ok {
return fail("the lease's expiry was not restored (" + why + "): the lease stays with mesh-grants")
}
if p.Lease == nil {
return fail("the part holds no hand on the lease here")
}
// The record first: the delete of removal-due is then heard as no removal (Handed is zero).
r.Handed, a.Step = time.Time{}, StepRestored
if err := p.writeLease(*r); err != nil {
return err
}
if err := p.Lease.DeleteHolder(); err != nil {
return fail("the key holder could not be deleted: " + err.Error())
}
if err := p.Store.Delete(StateLease, KeyRemovalDue); err != nil {
return fail("removal-due could not be deleted: " + err.Error())
}
r.Cut, a.Step = false, StepUncut
if err := p.writeLease(*r); err != nil {
return err
}
p.composeQuiet()
g, _, _ := p.Sealed()
fresh, err := p.readLease()
if err != nil || fresh.Open == nil {
return err
}
*r = fresh
a = r.Open
a.List = g.Digest
if err := p.writeLease(*r); err != nil {
return err
}
fallthrough
case StepUncut:
if !p.loaded(a.List) {
return nil
}
if started, why := p.service(CallServiceStart, r.Node); !started {
return fail("the controller's service on " + r.Node + " did not read back started (" + why + ")")
}
p.recordWarrant(a.Ask)
r.end(StepDone, "the lease is given back; the controller is started", p.now())
return p.writeLease(*r)
}
return nil
}
// controllerNode is the node placement places mesh-controller on (issue 774). Never the lease holder's host: that is
// the machine's hostname, and no node need be named so, so a seat verb addressed to it reaches nobody. None, or more
// than one, is refused by name and nothing is stopped.
func (p *Part) controllerNode() (string, string) {
if raw, found, err := p.Read.Get(ReadLease, LeaseKey); err == nil && found {
var h buscompose.LeaseHolder
if json.Unmarshal(raw, &h) == nil && h.HandedTo != "" {
return "", "the lease is handed to " + h.HandedTo + ", not the controller's: nothing is stopped"
}
}
in := ReadInputs(p.Read, &p.cache)
var nodes []string
for node, modules := range in.Assigned {
if slices.Contains(modules, Controller) {
nodes = append(nodes, node)
}
}
slices.Sort(nodes)
switch len(nodes) {
case 0:
return "", "no node places " + Controller + ", so the controller's node is not known: nothing is stopped"
case 1:
return nodes[0], ""
}
return "", "two or more nodes place " + Controller + " (" + strings.Join(nodes, ", ") + "), so the controller's node is not one: nothing is stopped"
}
// service asks the node seat's verb of the controller's unit and reads it back with status: stopped for a stop, started
// for a start.
func (p *Part) service(verb, node string) (bool, string) {
unit := map[string]any{"unit": ControllerUnit}
if _, err := p.Call.Call(verb+"@"+node, unit); err != nil {
return false, err.Error()
}
raw, err := p.Call.Call(CallServiceStatus+"@"+node, unit)
if err != nil {
return false, "its status cannot be read: " + err.Error()
}
var st struct {
Active string `json:"ActiveState"`
}
_ = json.Unmarshal(unwrap(raw), &st)
if verb == CallServiceStart {
return st.Active == "active", "it reads " + st.Active
}
return st.Active == "inactive" || st.Active == "failed", "it reads " + st.Active
}
// loaded says the bus node's have names the list of that digest loaded from the grants part's key.
func (p *Part) loaded(digest string) bool {
l := ReadInputs(p.Read, &p.cache).Loaded
return digest != "" && l != nil && l.Source == delivery.GrantsUserListSource && l.Digest == digest
}
// setExpiry asks the bus module's tool to set the lease bucket's expiry, and reads it back from the bucket.
func (p *Part) setExpiry(maxAge string, a LeaseAct) (bool, string) {
raw, err := p.Call.Call(CallBucketExpiry, map[string]any{"bucket": buscompose.LeaseBucket, "max_age": maxAge,
"why": a.Act + ": " + a.Why, "warrant": a.Ask})
if err != nil {
return false, err.Error()
}
var set struct {
Now string `json:"now"`
}
_ = json.Unmarshal(unwrap(raw), &set)
if set.Now != maxAge {
return false, "the tool answered " + set.Now
}
if p.Lease != nil {
got, err := p.Lease.Expiry()
want, _ := time.ParseDuration(strings.Replace(maxAge, "none", "0s", 1))
if err != nil || got != want {
return false, fmt.Sprintf("the bucket reads back %s (%v)", got, err)
}
}
return true, ""
}
// putRemovalDue writes removal-due with its time to live.
func (p *Part) putRemovalDue(ttl time.Duration) error {
s, ok := p.Store.(TTLStore)
if !ok {
return errors.New("the part's store writes no time to live here")
}
raw, _ := json.Marshal(map[string]any{"due": p.now().Add(ttl), "written": p.now()})
_, err := s.PutTTL(StateLease, KeyRemovalDue, raw, ttl)
return err
}
// saySeat says one of the seat's events; a failure is logged, never the end of the act.
func (p *Part) saySeat(event string, body any) {
if p.SaySeat == nil {
return
}
if err := p.SaySeat(event, body); err != nil && p.Logf != nil {
p.Logf("the event %s could not be said: %v", event, err)
}
}
// leaseStep is the pass's part in the lease write: the open act asked or advanced, the lease watched once handed, and
// the removal run when its marker was heard.
func (p *Part) leaseStep(res *Result) {
for _, step := range []func() error{p.raiseLease, p.advanceLease, p.watchLease} {
if err := step(); err != nil {
res.Errors = append(res.Errors, err.Error())
}
}
if p.removalHeard.Swap(false) {
if err := p.removal(); err != nil {
res.Errors = append(res.Errors, err.Error())
}
}
}
// RemovalDueHeard is the follower's: the marker of removal-due was heard (its delete, by expiry or by a hand-back).
func (p *Part) RemovalDueHeard() { p.removalHeard.Store(true) }
// watchLease says lease-altered once when the key or the bucket's expiry differs from what the part wrote (ADR 0582
// clause 4); a hand-back under way is the part's own change and is not watched.
func (p *Part) watchLease() error {
r, err := p.readLease()
if err != nil || r.Handed.IsZero() || r.Altered != "" || (r.Open != nil && r.Open.Act == ActLeaseHandBack && r.Open.Step != StepAsked) {
return err
}
var what []string
raw, found, err := p.Read.Get(ReadLease, LeaseKey)
var h buscompose.LeaseHolder
switch {
case err != nil:
return nil // not read is no change: the next pass reads it again
case !found:
what = append(what, "the key holder is gone")
case json.Unmarshal(raw, &h) != nil || h.HandedTo != Module || h.Ask != r.Ask:
what = append(what, "the key holder names "+firstOf(h.HandedTo, h.Instance, "something else")+", not mesh-grants by "+r.Ask)
}
if p.Lease != nil {
if d, err := p.Lease.Expiry(); err == nil && d != 0 {
what = append(what, "the lease's bucket expires after "+d.String())
}
}
if len(what) == 0 {
return nil
}
r.Altered = strings.Join(what, "; ")
p.saySeat(EventLeaseAltered, map[string]any{"altered": r.Altered, "ask": r.Ask, "at": p.now()})
return p.writeLease(r)
}
func firstOf(s ...string) string {
for _, x := range s {
if x != "" {
return x
}
}
return ""
}
// removal is the day-later removal, started by the marker of removal-due alone (ADR 0582 clause 6): when the lease still
// names mesh-grants, the controller's consumer of DEAD_LETTER_NOTICES and the stream ASSIGNMENTS are removed, each read
// back by the bus module's tool and recorded as a hand act. A failure is said as removal-failed and removal-due is
// written again with a day.
func (p *Part) removal() error {
r, err := p.readLease()
if err != nil || r.Handed.IsZero() || !r.Removed.IsZero() {
return err // handed back (a hand-back deletes the key), or done already
}
if raw, found, err := p.Read.Get(ReadLease, LeaseKey); err != nil || !found {
return p.removalFailed(r, "the lease cannot be read, so it is not known to name mesh-grants")
} else if away, _ := buscompose.LeaseHandedAway(raw, true, nil); !away {
return p.removalFailed(r, "the lease no longer names mesh-grants")
}
why := "a day after the lease write " + r.Ask + " (ADR 0582 clause 6)"
var failed []string
if raw, err := p.Call.Call(CallConsumerRemove, map[string]any{"stream": buscompose.DeadLetterNoticesStream,
"consumer": buscompose.ControllerName, "why": why}); err != nil && !strings.Contains(err.Error(), "has no consumer") {
failed = append(failed, "the consumer controller of DEAD_LETTER_NOTICES: "+err.Error())
} else if err == nil && !removed(raw) {
failed = append(failed, "the consumer controller of DEAD_LETTER_NOTICES does not read back removed")
} else {
p.recordHandAct("removed the controller's consumer of DEAD_LETTER_NOTICES", why)
}
if raw, err := p.Call.Call(CallStreamRemove, map[string]any{"stream": buscompose.AssignmentsStream, "why": why}); err != nil {
failed = append(failed, "the stream ASSIGNMENTS: "+err.Error())
} else if !removed(raw) {
failed = append(failed, "the stream ASSIGNMENTS does not read back removed")
} else {
p.recordHandAct("removed the stream ASSIGNMENTS", why)
}
if len(failed) > 0 {
return p.removalFailed(r, strings.Join(failed, "; "))
}
r.Removed = p.now()
return p.writeLease(r)
}
// removed reads a removal tool's answer: removed now, or not there to remove.
func removed(raw json.RawMessage) bool {
var a struct {
Removed bool `json:"removed"`
Present *bool `json:"present"`
}
_ = json.Unmarshal(unwrap(raw), &a)
return a.Removed || (a.Present != nil && !*a.Present)
}
func (p *Part) removalFailed(r LeaseRecord, why string) error {
p.saySeat(EventRemovalFailed, map[string]any{"failed": why, "ask": r.Ask, "at": p.now()})
if err := p.putRemovalDue(RemovalAfter); err != nil {
return fmt.Errorf("the removal failed (%s) and removal-due could not be written again: %v", why, err)
}
return fmt.Errorf("the day-later removal failed, tried again in a day: %s", why)
}
// recordHandAct records one removal in the hand-act log; said, never fatal.
func (p *Part) recordHandAct(what, why string) {
if p.Call == nil {
return
}
if _, err := p.Call.Call(CallRecordHandAct, map[string]any{"what": what, "why": why, "cause": "lease-removal"}); err != nil && p.Logf != nil {
p.Logf("%s, and the hand act could not be recorded: %v", what, err)
}
}
// RemovalDue is the tool grants_removal_due: removal-due written again with a short time to live, so its marker starts
// the removal (the recovery for a marker missed while the part was down). Recorded as a hand act.
func (p *Part) RemovalDue(caller, why, ttl string) (any, error) {
if strings.TrimSpace(why) == "" {
return nil, errors.New("grants_removal_due needs a why, kept with the hand act")
}
d, err := time.ParseDuration(ttl)
if err != nil || d < time.Second || d > RemovalAfter {
return nil, fmt.Errorf("ttl %q is no time to live between 1s and %s", ttl, RemovalAfter)
}
r, err := p.readLease()
if err != nil {
return nil, err
}
if r.Handed.IsZero() || !r.Removed.IsZero() {
return nil, errors.New("no removal is due: the lease is not handed to mesh-grants, or the removal ran")
}
if err := p.putRemovalDue(d); err != nil {
return nil, err
}
p.recordHandAct("wrote removal-due again with "+d.String()+" to live", why+" (by "+caller+")")
return map[string]any{"written": KeyRemovalDue, "ttl": d.String(), "says": "its marker starts the removal when it expires"}, nil
}
// LeaseState answers where the lease write stands, for grants_lease_check.
func (p *Part) LeaseState() (LeaseRecord, error) { return p.readLease() }
modules/mesh-grants/internal/grants/memberships.go
+3 -16
View File
@@ -14,9 +14,8 @@ import (
"git.novox.be/novox/mesh-sdk/go/membership"
)
// Memberships are published for real from this milestone (the operator, 2026-10-25 22:31): since release 6 nobody
// publishes them, so a module placed since cannot serve its seat or reach a new state. The user list, the assertions and
// the grants stay in shadow until take-over.
// Memberships are published for real (the operator, 2026-10-25 22:31): since release 6 nobody else publishes them, so a
// module placed since could not serve its seat or reach a new state without them.
// MembershipGaps are what a membership carries that the part cannot compose from the bus.
var MembershipGaps = []Gap{
@@ -24,14 +23,6 @@ var MembershipGaps = []Gap{
{Input: "membership: root-free machines", Why: "asked of the controller's root-free verb at each pass and kept for that pass alone; a machine it does not call free (an error included) is composed as not free, and the pass says why"},
}
// Publisher says a membership the part wrote to its state memberships on its subject (membership.Subject), as a plain
// message, for a runtime that follows the subject alone. The state is where a membership is published and read back
// (ADR 0571 clause 7); the subject is a notice that depends on no stream, so the assignments stream's removal at the
// lease write (research 072 row 33) stops nothing. A refusal is said and the membership stays written.
type Publisher interface {
PublishMembership(node, module string, body []byte) error
}
// Membership is one module's membership on one node, as composed now, and what became of its publishing.
type Membership struct {
Node string `json:"node"`
@@ -76,11 +67,7 @@ const (
// its key in the part's own state memberships, found when it is there), carrying that one's private network and
// received contributions. It answers m as composed and published, or why not. held is every holding the memberships
// composed now carry (Holdings.ByMemberships). Pure: the caller writes and notifies.
func publish(p Publisher, was []byte, found bool, m Membership, h Holdings, held map[string]bool) (Membership, bool) {
if p == nil {
m.Published = "not published: no publisher here (only the part's process publishes)"
return m, false
}
func publish(was []byte, found bool, m Membership, h Holdings, held map[string]bool) (Membership, bool) {
if found {
// A value of the state is the bare membership, or the wrapper written before 2026-10-30 (mesh-sdk's one codec);
// one that is neither is no last membership, and m is a first one.
modules/mesh-grants/internal/grants/part.go
+20 -61
View File
@@ -9,7 +9,6 @@ import (
"sort"
"strings"
"sync"
"sync/atomic"
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
@@ -23,18 +22,16 @@ const (
StateUserList = "user-list"
StateMemberships = "memberships"
StateAssertions = "assertions"
StateTakeover = "takeover"
// StateGrants is the part's grants by module and fingerprint, the key layout of the controller's (grants.go).
StateGrants = delivery.PartGrantsState
KeyList = "list"
KeySwitch = "switch"
// KeyPass is the last pass's result, and KeyAsked a tool's ask to compose again now, both in StateUserList.
KeyPass = "pass"
KeyAsked = "asked"
)
// States is every state the part keeps.
var States = []string{StateAccounts, StateAssertions, StateGrants, StateLease, StateMemberships, StateTakeover, StateUserList, StateWithheld}
var States = []string{StateAccounts, StateAssertions, StateGrants, StateMemberships, StateUserList, StateWithheld}
// The events the part says.
const (
@@ -63,7 +60,7 @@ type Emitter interface {
Emit(event string, body any) error
}
// UserListState is the value of `user-list` in shadow: the list the grants part composed (users and grants, no hashes),
// UserListState is the value of `user-list`: the list the grants part composed (users and grants, no hashes),
// what the bus loaded, and how they differ. Inputs is the digest of all it says but At: a value is written again only
// when it changes.
type UserListState struct {
@@ -140,7 +137,7 @@ type Result struct {
}
// NothingTakesEffect is what every composition says it did on the bus beyond its own state and the memberships.
const NothingTakesEffect = "memberships (in its own state memberships, each said on its subject, unless one would leave a seat a placed module claims with no holder, or drop a capability its holder still asks for, which stays a shadow difference), and each bucket, log or stream a manifest declares that the bus lacks, created once with the manifest's limits (the operator, 2026-10-26 08:54); the user list, the grants and any change to a stream the bus holds stay in shadow until take-over"
const NothingTakesEffect = "memberships (in its own state memberships, unless one would leave a seat a placed module claims with no holder, or drop a capability its holder still asks for, which stays a shadow difference), each bucket, log or stream a manifest declares that the bus lacks, created once with the manifest's limits (the operator, 2026-10-26 08:54), the user list the bus node follows, the grants and the account consumers; a change to a stream the bus holds is only said"
// Part is the grants part: it reads, calls and writes through these, and nothing else.
type Part struct {
@@ -148,39 +145,21 @@ type Part struct {
Call Caller
Store Store
Emit Emitter
// Publish publishes memberships: the process's connection; nil in the tools, which ask the process instead.
Publish Publisher
// Create creates a declared stream the bus lacks: the process's connection; nil in the tools, which ask the process.
Create Creator
// Asker asks the operator (take-over, hand-back): the process's client of the router; nil in the tools, which write
// the request and let the process ask it.
Asker Asker
// Logf logs what no state or condition says (an error of acting on an answer): the process's; nil in the tools.
Logf func(string, ...any)
// Objects is the part's object bucket of sealed lists: the process's connection; nil in the tools.
Objects Objects
// Consumers creates an account's durable consumer from the lease write (consumers.go): the process's connection;
// nil in the tools, which compare only.
// Consumers creates an account's durable consumer (consumers.go): the process's connection; nil in the tools, which
// compare only.
Consumers ConsumerCreator
// Bus is where the nodes reach the bus, which each wanted account carries (accounts.go): the process's own credential's;
// empty in the tools, which write no account.
Bus BusAddress
// Say says a condition to the operator: the process's; nil in the tools.
Say func(asks.Condition) error
// Forget takes away the record the asker kept of an ask it failed to publish; Nonce is a test's id suffix.
Forget func(id string) error
Nonce func() string
// Lease is the process's hand on the controller's lease (leasewrite.go); nil in the tools, which ask the process.
Lease LeaseBus
// SaySeat says the seat mesh-grants' events (lease-handed, lease-altered, removal-failed): the process's; nil in the
// tools.
SaySeat SeatSayer
// deciding is set while a take-over is decided, or a lease act composes, so the composition it makes asks and
// reconciles nothing.
deciding atomic.Bool
// removalHeard is set when the marker of removal-due was heard, and taken by the next pass (leasewrite.go).
removalHeard atomic.Bool
Now func() time.Time
Now func() time.Time
cache Manifests
mu sync.Mutex
@@ -197,6 +176,11 @@ type Part struct {
// differSaid is the account consumers' differences last said as a condition, once differKnown (consumers.go).
differSaid string
differKnown bool
// behindSaid is the account consumers far behind last said as a condition, once behindKnown; behindSeen is each one
// found behind at the pass before, so one look at a burst raises nothing (consumers.go).
behindSaid string
behindKnown bool
behindSeen map[string]bool
// refusedSaid is the refused principals last said as a condition, once refusedKnown (refused.go).
refusedSaid string
refusedKnown bool
@@ -257,12 +241,10 @@ func (p *Part) Compose() Result {
// the trunk names, so the list grants a build's subjects before the want moves to it (issue 774).
recs, _ := Records(in)
listRecs, left := ListRecords(in)
// From the lease write's cut, the controller's write on the lease is taken from the list (ADR 0582 clause 3).
listRecs.LeaseCut = p.leaseCut()
holdings := HoldingsOf(in)
res.RootNotFree = WithRootFree(&listRecs, p.Call)
recs.RootFree = listRecs.RootFree
st := UserListState{V: 1, Mode: p.SwitchState(), LeftOut: left, Loaded: in.Loaded, BusNode: in.BusNode,
st := UserListState{V: 1, Mode: InForce, LeftOut: left, Loaded: in.Loaded, BusNode: in.BusNode,
Gaps: append(append([]Gap(nil), in.Gaps...), MembershipGaps...), At: p.now()}
if st.LeftOut == nil {
st.LeftOut = []Gap{}
@@ -299,7 +281,7 @@ func (p *Part) Compose() Result {
} else {
res.Revision, res.Written = rev, true
p.say(&res, EventUserList, map[string]any{"digest": st.Digest, "revision": rev, "users": st.Users,
"differing": len(st.Compare.Differing()), "mode": Shadow})
"differing": len(st.Compare.Differing()), "mode": InForce})
}
}
}
@@ -312,8 +294,8 @@ func (p *Part) Compose() Result {
p.writeWithheld(&res, in, recs)
}
list, n, leftOut, dropped, err := ComposeList(listRecs, in.Hashes, refused, kept)
// The grants the assigner reads once the lease is handed away, in shadow until then (ADR 0547 §4, §5); after take-over
// one that adds a permission only once the bus loaded a list that carries it (issue 774).
// The grants the assigner reads (ADR 0547 §4, §5): one that adds a permission only once the bus loaded a list that
// carries it (issue 774).
if p.Store != nil {
inForce, unknown := p.loadedContents(in, loaded, loadErr)
p.writeGrants(&res, in, listRecs, users, inForce, unknown)
@@ -322,7 +304,7 @@ func (p *Part) Compose() Result {
if err != nil {
res.Errors = append(res.Errors, "the user list could not be composed whole: "+err.Error())
} else {
// The users the bus loaded: what a first list after take-over must keep where the part has not yet written one.
// The users the bus loaded: what a first list must keep where the part has not yet written one.
var bus, lost []string
if st.Compare.Unread == "" {
bus = names(loaded)
@@ -340,17 +322,16 @@ func (p *Part) Compose() Result {
// The memberships are written to the state alone and never said on their subject, which nothing reads any more
// (novox/hq ADR 0571 clause 10, issue 774; unconditional since the controller's end, issue 1126).
for _, key := range sortedKeys(ms) {
// The last membership is the state's own (ADR 0571 clause 7), never the assignments stream's, which the bus
// module's tools remove at the lease write (research 072 row 33): a last read there would freeze every membership
// once it is gone.
// The last membership is the state's own (ADR 0571 clause 7), never the assignments stream's, which goes with the
// controller's leftovers (novox/hq ADR 0590 §6).
was, _, found, err := p.Store.Get(StateMemberships, key)
if err != nil {
res.NotPublished = append(res.NotPublished, key+": not published: the membership last published cannot be read from its own "+StateMemberships+": "+err.Error())
continue
}
m, now := publish(p.Publish, was, found, ms[key], holdings, seated)
m, now := publish(was, found, ms[key], holdings, seated)
// Published is written to the state, where every runtime reads it (process.FollowMembership); only one
// published, so a membership held back never reaches a reader. The subject is a notice after it.
// published, so a membership held back never reaches a reader.
if now {
body := membership.EncodeGrantsMembership(m.Membership)
if _, err := p.Store.Put(StateMemberships, key, body); err != nil {
@@ -406,31 +387,9 @@ func (p *Part) Compose() Result {
}
}
p.consumers(&res, recs, ask.Consumer)
if _, _, found, err := p.Store.Get(StateTakeover, KeySwitch); err == nil && !found {
data, _ := json.Marshal(Switch{V: 1, State: Shadow, Acts: []Act{}})
if _, err := p.Store.Put(StateTakeover, KeySwitch, data); err != nil {
res.Errors = append(res.Errors, fmt.Sprintf("its own %s cannot be written: %v", StateTakeover, err))
}
}
if !p.deciding.Load() {
if err := p.RaiseRequested(res.UserList); err != nil {
res.Errors = append(res.Errors, err.Error())
}
if err := p.Reconcile(); err != nil {
res.Errors = append(res.Errors, err.Error())
}
p.leaseStep(&res)
}
return res
}
// composeQuiet composes as a step of an act: asking and reconciling nothing.
func (p *Part) composeQuiet() Result {
p.deciding.Store(true)
defer p.deciding.Store(false)
return p.Compose()
}
// Last is the result of the last pass this part made, kept under the lock alone.
func (p *Part) Last() Result {
p.mu.Lock()
modules/mesh-grants/internal/grants/records.go
+2 -3
View File
@@ -36,10 +36,9 @@ func records(in Inputs, union bool) (buscompose.Records, []Gap) {
for _, own := range catalogue.SeatsWithAProtocol() {
seats[own.Name] = catalogue.SeatDeclaration{Name: own.Name, Scope: own.Scope, Accepts: own.Accepts, Emits: own.Emits, Serves: own.Serves}
}
// The enrolment users, from the seat mesh-enrolment's keys, and from the lease write through its redeem alone (ADR 0587
// §4, §6).
// The enrolment users, from the seat mesh-enrolment's keys, enrolling through its redeem alone (ADR 0587 §4, §6).
out := buscompose.Records{Assigned: map[string][]buscompose.Declared{}, People: map[string][]string{},
Interchangeable: map[string]bool{}, Enrolling: slices.Clone(in.Enrolling), LeaseHandedAway: true,
Interchangeable: map[string]bool{}, Enrolling: slices.Clone(in.Enrolling),
// And each node's logins (ADR 0583 §1): both while it rotates.
NodeLogins: in.NodeLogins}
values := func(provider, setting string) []any {
modules/mesh-grants/internal/grants/rootfree.go
+10 -27
View File
@@ -18,30 +18,20 @@ import (
const CallRootFree = "seat:node-engine.root-free"
// The condition events the part hears (never a timer): a machine's root-free word changes with the raising and the
// clearing of its `root-not-free` condition. They are the controller seat's until the operator's take-over, and the
// condition keeper's (mesh-conditions) from it: the part hears only the seat the take-over key names, never both (novox/hq
// ADR 0564 §9, issue 1952 G1).
// clearing of its `root-not-free` condition. They are the condition keeper's (mesh-conditions) alone, the one keeper
// since the controller ended by its unassignment (novox/hq ADR 0590 §3, which ends ADR 0564 §9's selection by the
// take-over key).
const (
EventConditionRaised = "mesh-controller.condition-raised"
EventConditionCleared = "mesh-controller.condition-cleared"
EventKeeperRaised = health.KeeperModule + ".condition-raised"
EventKeeperCleared = health.KeeperModule + ".condition-cleared"
EventKeeperRaised = health.KeeperModule + ".condition-raised"
EventKeeperCleared = health.KeeperModule + ".condition-cleared"
// KindRootNotFree is the condition kind whose events wake a pass.
KindRootNotFree = "root-not-free"
)
// ReadTakeover is the condition keeper's take-over key's state (`switch`): which seat's condition events the part hears.
const ReadTakeover = health.KeeperModule + "." + health.TakeoverState
// HeardFrom is the seat a condition event heard comes from, by its name (`<seat>.<event>`) or its bus subject
// (`mesh.seat.<seat>.event.<event>`); empty for any other.
func HeardFrom(event string) string {
for _, seat := range []string{health.KeeperModule, "mesh-controller"} {
if strings.HasPrefix(event, seat+".") || strings.HasPrefix(event, "mesh.seat."+seat+".event.") {
return seat
}
}
return ""
// FromKeeper says a condition event heard is the keeper's, by its name (`<seat>.<event>`) or its bus subject
// (`mesh.seat.<seat>.event.<event>`).
func FromKeeper(event string) bool {
return strings.HasPrefix(event, health.KeeperModule+".") || strings.HasPrefix(event, "mesh.seat."+health.KeeperModule+".event.")
}
// EventAccountIssued is the vault's word that it minted an account (novox/hq ADR 0561 §1): its hash is in the vault's
@@ -49,20 +39,13 @@ func HeardFrom(event string) string {
const EventAccountIssued = "mesh-vault." + buscompose.AccountIssuedEvent
// Consumes is every event the part hears, as its manifest's consumes name them.
//
// Not the controller's condition events: the keeper's alone (novox/hq ADR 0564 §9, issue 774).
var Consumes = []string{EventKeeperRaised, EventKeeperCleared, EventAccountIssued, EventBucketExpirySet}
var Consumes = []string{EventKeeperRaised, EventKeeperCleared, EventAccountIssued}
// IsAccountIssued says an event heard is the vault's account-issued, by its name or its bus subject.
func IsAccountIssued(event string) bool {
return event == EventAccountIssued || event == buscompose.AccountIssuedSubject
}
// EventSubject is the bus subject of a controller seat event named `mesh-controller.<event>`.
func EventSubject(event string) string {
return "mesh.seat.mesh-controller.event." + strings.TrimPrefix(event, "mesh-controller.")
}
// IsRootNotFree says a condition event's body names the `root-not-free` kind (its `kind`, or the last part of its
// `key`, `<scope>.<id>.<kind>`). An unreadable body wakes a pass too: a pass is cheap, a missed change is not.
func IsRootNotFree(body []byte) bool {
modules/mesh-grants/internal/grants/sealed.go
+16 -96
View File
@@ -14,13 +14,12 @@ import (
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// **The list the bus node loads from take-over** (novox/hq ADR 0547 clause 7, issue 1945). Each pass composes the bus's
// user list with the vault's hashes (the accounts block, buscompose.ComposeAccounts, as the bus module's file holds it),
// seals it to the bus node's key (the key the node says in its have of the bus module) and puts it as one object of the
// part's object bucket, named by its digest; then writes the key `user-list.sealed` naming it, with the revision, the user
// count, the users left out for want of a hash, and which list the bus node follows (the switch, clause 8). The object
// and the key are written in shadow too: the bus node reads the key and follows the controller's list until the key says
// FollowGrants, so a take-over has a list to load at once. Only the process writes them (Objects); a tool asks it.
// **The list the bus node loads** (novox/hq ADR 0547 clause 7, issue 1945). Each pass composes the bus's user list with
// the vault's hashes (the accounts block, buscompose.ComposeAccounts, as the bus module's file holds it), seals it to the
// bus node's key (the key the node says in its have of the bus module) and puts it as one object of the part's object
// bucket, named by its digest; then writes the key `user-list.sealed` naming it, with the revision, the user count and
// the users left out for want of a hash. The key always says FollowGrants: the bus node follows this list alone since the
// controller ended by its unassignment (ADR 0590, issue 1126). Only the process writes them (Objects); a tool asks it.
// KeySealed is the key of the state user-list the bus node reads (delivery.GrantsUserListKey).
const KeySealed = delivery.GrantsUserListKey
@@ -81,23 +80,14 @@ func (p *Part) Sealed() (delivery.GrantsUserList, uint64, bool) {
return g, rev, true
}
// followOf is which list the switch says the bus node follows.
func followOf(state string) string {
if state == TookOver {
return delivery.FollowGrants
}
return delivery.FollowController
}
// writeSealed puts the list sealed to the bus node and writes the key, each only when what it says changed. It says in
// res what it wrote, or why it wrote nothing.
func (p *Part) writeSealed(res *Result, list []byte, users int, leftOut, dropped []string, in Inputs, bus []string, lost []string, composed []User) {
cur, _, had := p.Sealed()
next := cur
next.Follow = followOf(p.SwitchState())
// The bus node follows this list alone (ADR 0571 clause 8): always since the controller ended by its unassignment,
// with no lease written (issue 1126).
next.LeaseHanded = true
// The bus node follows this list alone (ADR 0571 clause 8): always since the controller ended by its unassignment
// (novox/hq ADR 0590, issue 1126).
next.Follow = delivery.FollowGrants
key := ""
if in.Loaded != nil {
key = in.Loaded.Key
@@ -106,16 +96,16 @@ func (p *Part) writeSealed(res *Result, list []byte, users int, leftOut, dropped
why, held := "", ""
switch {
case p.Objects == nil:
// The tools: the process writes; the follow alone may change here, from a tool's view of the switch, and is not.
// The tools: the process writes.
res.Sealed = sealedSays(cur, had, "")
return
case in.BusNode == "":
why = "no node runs the bus's module, so the list is sealed to nobody"
case key == "":
why = "the bus node's have of the bus module names no key yet (its node-engine says it from this release)"
case next.Follow == delivery.FollowGrants && cur.Digest != digest && p.holdsBack(in, leftOut, dropped, bus, lost, digest) != "":
// After take-over the list in force is never replaced by one composed past a failed read, or one that drops a user
// of it for want of a hash: that user's module would be locked out of the bus (review of the switch).
case cur.Digest != digest && p.holdsBack(in, leftOut, dropped, bus, lost, digest) != "":
// The list in force is never replaced by one composed past a failed read, or one that drops a user of it for want
// of a hash: that user's module would be locked out of the bus (review of the switch).
why = p.holdsBack(in, leftOut, dropped, bus, lost, digest)
held = why
p.keepHashHold(res, digest)
@@ -198,7 +188,7 @@ func (p *Part) sayHeld(why string) {
}
}
// HeldCondition is the key of the warning raised while a list in force is held back after take-over.
// HeldCondition is the key of the warning raised while a list in force is held back.
const HeldCondition = "mesh-grants.user-list-held"
func sameSealed(a, b delivery.GrantsUserList) bool {
@@ -231,48 +221,6 @@ func sealedSays(g delivery.GrantsUserList, had bool, why string) *SealedState {
return s
}
// syncFollow writes the switch into the key at once, so the bus node turns on the operator's yes, not at the next pass.
func (p *Part) syncFollow(state string) error {
cur, _, had := p.Sealed()
if had && cur.Follow == followOf(state) {
return nil
}
if !had {
cur = delivery.GrantsUserList{}
}
cur.Follow, cur.At = followOf(state), p.now()
data, err := delivery.EncodeGrantsUserList(cur)
if err != nil {
return err
}
_, err = p.Store.Put(StateUserList, KeySealed, data)
return err
}
// Effect is where the switch stands on the bus node: "switched" only once the bus node's loaded list is the grants part's
// written one (its source the part's key, its digest the written digest); otherwise what it waits for.
func (p *Part) Effect(state string, loaded *delivery.UserListWritten) string {
g, _, had := p.Sealed()
fromGrants := loaded != nil && loaded.Source == delivery.GrantsUserListSource
switch {
case state == TookOver && had && fromGrants && g.Digest != "" && loaded.Digest == g.Digest && loaded.Error == "":
return "switched: the bus node loaded the grants part's list " + short(g.Digest) + " (revision " + fmt.Sprint(g.Revision) + ")"
case state == TookOver && !had || state == TookOver && g.Digest == "":
return "recorded, not in effect: the grants part has written no sealed list for the bus node yet"
case state == TookOver && loaded == nil:
return "recorded, not in effect: the bus module's have names no loaded list"
case state == TookOver && loaded.Unwatched != "" && !fromGrants:
return "recorded, not in effect: the bus node does not watch the grants part's key: " + loaded.Unwatched
case state == TookOver && loaded.Error != "":
return "recorded, not in effect: the bus node could not load the grants part's list: " + loaded.Error
case state == TookOver:
return "recorded, not in effect yet: the bus node loaded " + short(loaded.Digest) + " from " + loaded.Source + ", the grants part wrote " + short(g.Digest)
case fromGrants:
return "handed back, not in effect yet: the bus node still follows the grants part's list"
}
return "shadow: the bus node follows the controller's list"
}
func short(d string) string {
d = strings.TrimPrefix(d, "sha256:")
if len(d) > 12 {
@@ -281,36 +229,8 @@ func short(d string) string {
return d
}
// followReasons is what a take-over waits for on the sealed list itself: a list written for the bus node, of the list
// composed now, and the loaded list granting the bus node its read (the controller's composer carries ADR 0547 clause 7's
// grants from the operator's second exception; until its delivery the bus node could not read the first list).
func (p *Part) followReasons(res Result, in Inputs) []string {
var out []string
g, _, had := p.Sealed()
switch {
case !had || g.Digest == "":
why := "no list is written"
if res.Sealed != nil && res.Sealed.Why != "" {
why = res.Sealed.Why
}
out = append(out, "the grants part has written no sealed list for the bus node: "+why)
case in.Loaded == nil || in.Loaded.Key == "" || g.SealedTo != delivery.SealingKeyID(in.Loaded.Key):
out = append(out, "the sealed list is not sealed to the key the bus node says now")
}
if in.BusNode != "" {
node := "node." + in.BusNode
for _, d := range res.UserList.Compare.Differ {
if d.User == node && slices.ContainsFunc(d.PublishAdded, func(s string) bool { return strings.Contains(s, delivery.GrantsUserListBucket) }) {
out = append(out, "the loaded list does not grant "+node+" reading the grants part's sealed list (ADR 0547 clause 7): "+
"the controller's composer carries it from its delivery of the operator's second exception")
}
}
}
return out
}
// KeyWrittenUsers is the key of the state user-list naming the users of the list the key `sealed` names: what a list
// composed after take-over must keep.
// composed must keep.
const KeyWrittenUsers = "sealed-users"
// KeyWrittenCredentials is the key of the state user-list naming, per user of the list in force, which password its hash
@@ -371,7 +291,7 @@ func listUsers(list []byte) []string {
return out
}
// holdsBack says why a list composed now may not replace the one in force after take-over: a read failed this pass, a
// holdsBack says why a list composed now may not replace the one in force: a read failed this pass, a
// user of the list in force is left out for want of a hash, or the list would leave a seat the list in force holds with
// no holder while a placed module still claims it (lost, Holdings.LastHolders: the membership's guard, issue 1945; the
// loaded list not read holds it too, since that cannot be ruled out). The list in force is the one the part wrote, by the users
modules/mesh-grants/internal/grants/takeover.go
+5 -552
View File
@@ -1,556 +1,9 @@
package grants
import (
"git.novox.be/novox/mesh-sdk/go/handacts"
"crypto/rand"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/asker"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The switch between the controller's user list and the grants part's (ADR 0547 clause 8). The operator's word is a
// warrant: the operator's answer to an ask on a channel that proves who answered, never a caller's name, since an
// operator's session and every agent's call as the same machine account (the operator, 2026-10-26).
// Shadow is the switch's state until take-over: the bus node follows the controller's list. TookOver is its state after.
const (
Shadow = "shadow"
TookOver = "took-over"
)
// KeyRequest is the key in StateTakeover where a tool leaves its request for the process to ask.
const KeyRequest = "request"
// The two acts the operator may warrant.
const (
ActTakeOver = "take-over"
ActHandBack = "hand-back"
)
// Switch is the value of `takeover`: which list the bus node follows, every act on the switch and the ask open now.
type Switch struct {
V int `json:"v"`
State string `json:"state"`
// Open is the id of the ask the operator has not answered yet, empty when none.
Open string `json:"open,omitempty"`
// Refusing are the acts whose refusal condition was said and not cleared yet.
Refusing []string `json:"refusing,omitempty"`
Acts []Act `json:"acts"`
}
// Act is one take-over or hand-back asked, answered, done or refused.
type Act struct {
Act string `json:"act"`
By string `json:"by"`
Why string `json:"why,omitempty"`
At time.Time `json:"at"`
// Ask is the router's ask of the act; Said what became of it: asked, switched, declined, expired, refused.
Ask string `json:"ask,omitempty"`
Said string `json:"said,omitempty"`
Refused string `json:"refused,omitempty"`
}
// Request is what a tool leaves for the process to ask: written once per act, marked raised when asked.
type Request struct {
ID string `json:"id"`
Act string `json:"act"`
By string `json:"by"`
Why string `json:"why,omitempty"`
At time.Time `json:"at"`
Raised bool `json:"raised,omitempty"`
// RaisedAt is when it was asked, and Failed why asking it failed (the ask is then closed as refused).
RaisedAt time.Time `json:"raised_at,omitempty"`
Failed string `json:"failed,omitempty"`
// Adds is the digest of the rights the grants part's list added when the take-over was asked (Comparison.AddedDigest):
// the rights the ask named, which its yes answers (ADR 0562 §3).
Adds string `json:"adds,omitempty"`
}
// Asker is the client of the router the part asks through (mesh-sdk asker.Client).
type Asker interface {
Ask(a asks.Ask) error
// Where is the router's record of an ask: its state and its warrant, if any.
Where(id string) (string, *asks.Warrant, error)
// Take hands a warrant to act once (a warrant the process missed is caught up through it).
Take(w asks.Warrant, act func(asks.Option, asks.Warrant) error) (asker.Handled, error)
// Cancel takes an ask back at the router, so a copy the operator may still see ends there too.
Cancel(id string) error
}
// askMargin is how long past an ask's expiry the router is given to say it ended, before the part closes it itself;
// recordGrace how long a just-asked ask may be missing from the router's record.
const (
askMargin = 5 * time.Minute
recordGrace = time.Minute
)
// keptActs is how many acts the switch keeps, newest last.
const keptActs = 50
func (p *Part) readSwitch() (Switch, error) {
s := Switch{V: 1, State: Shadow}
raw, _, found, err := p.Store.Get(StateTakeover, KeySwitch)
if err != nil {
return s, err
}
if found {
_ = json.Unmarshal(raw, &s)
}
return s, nil
}
func (p *Part) writeSwitch(s Switch) error {
if len(s.Acts) > keptActs {
s.Acts = s.Acts[len(s.Acts)-keptActs:]
}
if s.Acts == nil {
s.Acts = []Act{}
}
data, _ := json.Marshal(s)
_, err := p.Store.Put(StateTakeover, KeySwitch, data)
return err
}
// SwitchState is which list the bus node follows now.
func (p *Part) SwitchState() string {
if s, err := p.readSwitch(); err == nil && s.State != "" {
return s.State
}
return Shadow
}
// TakeOver is the tool: it asks the operator to switch the bus node to the grants part's list. It writes the request,
// and the part's process asks it on the operator's channel; only the operator's yes (a warrant) runs the switch, after
// the refusals of Decide. Asking again while an ask is open is refused.
func (p *Part) TakeOver(caller, why string) (any, error) { return p.request(ActTakeOver, caller, why) }
// HandBack is the tool: it is answered as gone. The controller ended by its unassignment (issue 1126), so nothing
// writes its list and the grants part's list is the only one (ADR 0547 §11).
func (p *Part) HandBack(caller, why string) (any, error) {
return nil, errors.New("gone: the controller ended by its unassignment (issue 1126), so nothing writes its user list; " +
"the grants part's list is the only one and the user list is not handed back (ADR 0547 §11)")
}
// nonce is four hex digits that keep two asks of one second apart.
func (p *Part) nonce() string {
if p.Nonce != nil {
return p.Nonce()
}
var b [2]byte
_, _ = rand.Read(b[:])
return hex.EncodeToString(b[:])
}
func (p *Part) request(act, caller, why string) (any, error) {
s, err := p.readSwitch()
if err != nil {
return nil, fmt.Errorf("the switch cannot be read: %w", err)
}
a := Act{Act: act, By: caller, Why: why, At: p.now()}
if s.Open != "" {
return nil, fmt.Errorf("%s is already asked: the ask %s waits for the operator's answer", act, s.Open)
}
if act == ActHandBack && s.State == Shadow {
a.Refused = "nothing to hand back: the grants part is in shadow and never took over, so the bus node follows the controller's list already"
s.Acts = append(s.Acts, a)
_ = p.writeSwitch(s)
return nil, fmt.Errorf("%s", a.Refused)
}
if act == ActTakeOver && s.State == TookOver {
return nil, fmt.Errorf("the bus node follows the grants part's list already")
}
a.Ask = act + "-" + p.now().Format("20060102t150405") + "-" + p.nonce()
a.Said = "asked"
req, _ := json.Marshal(Request{ID: a.Ask, Act: act, By: caller, Why: why, At: a.At})
if _, err := p.Store.Put(StateTakeover, KeyRequest, req); err != nil {
return nil, fmt.Errorf("the request could not be written: %w", err)
}
s.Open = a.Ask
s.Acts = append(s.Acts, a)
if err := p.writeSwitch(s); err != nil {
return nil, fmt.Errorf("the act could not be recorded: %w", err)
}
// The process asks at its next pass: wake it.
_, _ = p.AskCompose(caller)
return map[string]any{"asked": a.Ask, "says": "the part's process asks the operator on their channel; only the operator's yes records the switch, and it takes effect when the bus node loads the grants part's list. grants_take_over_state shows where it stands"}, nil
}
// RaiseRequested is the process's: it asks the operator the request a tool left, once, naming the rights the pass's
// comparison c adds. Where the part has no asker (the tools) it does nothing.
func (p *Part) RaiseRequested(st UserListState) error {
if p.Asker == nil {
return nil
}
raw, rev, found, err := p.Store.Get(StateTakeover, KeyRequest)
if err != nil || !found {
if err != nil {
return fmt.Errorf("its own %s %s cannot be read: %v", StateTakeover, KeyRequest, err)
}
return nil
}
var r Request
if json.Unmarshal(raw, &r) != nil || r.Raised || r.Failed != "" || r.ID == "" {
return nil
}
if s, err := p.readSwitch(); err != nil || s.Open != r.ID {
return nil // an ask no longer open is not asked
}
ask, err := p.AskOf(r, st)
if err != nil {
return err
}
if r.Act == ActTakeOver {
r.Adds = st.Compare.AddedDigest()
}
if err := p.Asker.Ask(ask); err != nil {
// Closed as refused, and the half-made record the asker kept before publishing is taken away, so nothing stays open.
r.Failed = err.Error()
data, _ := json.Marshal(r)
_, _ = p.Store.Put(StateTakeover, KeyRequest, data)
if p.Forget != nil {
_ = p.Forget(r.ID)
}
// A publish that timed out may still have reached the router: taken back there, so a yes to it cannot land on an
// ask the part has forgotten.
if cerr := p.Asker.Cancel(r.ID); cerr != nil && p.Logf != nil {
p.Logf("the ask %s that could not be published was not taken back at the router: %v", r.ID, cerr)
}
if s, rerr := p.readSwitch(); rerr == nil {
_ = p.finish(s, Act{Act: r.Act, By: "operator", At: p.now(), Ask: r.ID, Said: "refused", Refused: "the operator could not be asked: " + err.Error()})
}
return fmt.Errorf("the operator could not be asked the %s: %v", r.Act, err)
}
r.Raised, r.RaisedAt = true, p.now()
data, _ := json.Marshal(r)
if _, err := p.Store.Put(StateTakeover, KeyRequest, data); err != nil {
return fmt.Errorf("the ask %s was asked and its request could not be marked: %v", r.ID, err)
}
_ = rev
return nil
}
// Reconcile closes an ask that can stay open for ever, once per pass while one is open: a warrant the process missed is
// caught up from the router's record, one that ended with none is closed, and one past its expiry and a margin is closed
// as expired by the part itself.
func (p *Part) Reconcile() error {
if p.Asker == nil {
return nil
}
s, err := p.readSwitch()
if err != nil || s.Open == "" {
return err
}
act := actOf(s.Open)
at := p.now()
for _, a := range s.Acts {
if a.Ask == s.Open {
at = a.At
break
}
}
state, w, err := p.Asker.Where(s.Open)
if err != nil {
return fmt.Errorf("the router's record of the ask %s cannot be read: %v", s.Open, err)
}
if w != nil {
h, herr := p.Asker.Take(*w, p.Warranted)
if h == "" && herr != nil {
// The part's own record of the ask could not be reached: nothing was taken, and the next pass tries again.
return fmt.Errorf("the operator's answer to %s could not be taken yet: %v", s.Open, herr)
}
p.Said(h, *w, herr)
if cur, rerr := p.readSwitch(); rerr == nil && cur.Open == s.Open {
_ = p.finish(cur, Act{Act: act, By: "operator", At: p.now(), Ask: s.Open, Said: string(w.Outcome)})
}
return nil
}
raw, _, found, _ := p.Store.Get(StateTakeover, KeyRequest)
var r Request
if found {
_ = json.Unmarshal(raw, &r)
}
switch {
case p.now().After(at.Add(asks.ApproveLasts + askMargin)):
return p.finish(s, Act{Act: act, By: "operator", At: p.now(), Ask: s.Open, Said: string(asks.OutcomeExpired)})
case state == "" && r.ID == s.Open && r.Raised && p.now().After(r.RaisedAt.Add(recordGrace)):
return p.finish(s, Act{Act: act, By: "operator", At: p.now(), Ask: s.Open, Said: "ended", Refused: "the router holds no record of the ask"})
}
return nil
}
func actOf(id string) string {
if strings.HasPrefix(id, ActHandBack+"-") {
return ActHandBack
}
return ActTakeOver
}
// Following says whether the bus node has loaded the grants part's list: "yes" when its loaded digest equals the part's,
// "not yet" when it loaded another, "unknown" when its have says none.
func (p *Part) Following() string {
loaded := ReadInputs(p.Read, &p.cache).Loaded
g, _, had := p.Sealed()
if loaded == nil || !had || g.Digest == "" {
return "unknown: the bus module's have names no loaded list, or the grants part wrote none"
}
if loaded.Source == delivery.GrantsUserListSource && loaded.Digest == g.Digest {
return "yes"
}
return "not yet: the bus node loaded another list"
}
// AskOf is the ask of a request, in the words the operator reads: what changes, why, what it risks; for a take-over every
// right the grants part's list adds by the user list st compares now, by user, or what a yes would be refused for
// (ADR 0562 §3, TakeOverWords). Its yes binds the act.
func (p *Part) AskOf(r Request, st UserListState) (asks.Ask, error) {
c := st.Compare
verb, headline, does, stays, expl := "grants_take_over", "Switch the bus's user list to the grants part?", "The switch is recorded; it takes effect when the bus node loads the grants part's list.",
"The controller's list stays in force.", "The grants part has composed the user list the bus holds today. Your yes records the switch; it takes effect when the bus node loads the part's list.\n"+
"If the lists differ other than by an added right, or a user has no hash in the vault, the switch is refused and you are told which."
if r.Act == ActHandBack {
verb, headline, does, stays, expl = "grants_hand_back", "Give the bus's user list back to the controller?", "The switch back is recorded; it takes effect when the bus node loads the controller's list.",
"The grants part's list stays in force.", "The bus follows the grants part's list now.\nThe controller still writes its list until it loses its lease, so the bus can follow it again."
}
details := "Asked by " + r.By + ": " + r.Why
act := asks.Act{"verb": verb, "module": "mesh-grants", "ask": r.ID}
if summary, points, list := TakeOverWords(c, st.Credentials); r.Act == ActTakeOver && summary != "" {
if len(Refusals(c, st.Credentials)) > 0 {
expl = summary + "\n" + strings.Join(points, "\n")
details = "Refused for:\n" + list + "\n" + details
} else {
expl = summary + " Your yes records the switch; it takes effect when the bus node loads the part's list.\n" + strings.Join(points, "\n")
details = "Added, by user:\n" + list + "\nA user added or dropped, a right taken away or a missing hash refuses the switch, naming them.\n" + details
}
act["adds"] = c.AddedDigest()
}
bind, err := asks.ActDigest(act)
if err != nil {
return asks.Ask{}, err
}
return asks.Ask{ID: r.ID, Headline: headline, Explanation: expl, Who: asks.Operator,
Options: []asks.Option{{ID: "yes", Label: "Yes, switch", Does: does, Level: asks.Approve, Binds: bind},
{ID: "no", Label: "No, leave it", Does: stays, Level: asks.Acknowledge}},
Expires: p.now().Add(asks.ApproveLasts - time.Minute), OnExpiry: stays + " Nothing changes.", About: "mesh-grants-" + r.Act,
Details: details}, nil
}
// Warranted is the process's act on the operator's answer (asker.Client.Listen hands it once, after checking the ask, the
// option and its level). A no is recorded; a yes runs the switch after the refusals, and a refusal is said to the operator.
func (p *Part) Warranted(o asks.Option, w asks.Warrant) error {
if isLeaseAsk(w.Ask) {
return p.leaseWarranted(o, w)
}
s, err := p.readSwitch()
if err != nil {
return err
}
act := ActTakeOver
if strings.HasPrefix(w.Ask, ActHandBack+"-") {
act = ActHandBack
}
if s.Open != w.Ask {
return fmt.Errorf("the warrant answers the ask %s, and the ask open is %q: nothing is switched", w.Ask, s.Open)
}
a := Act{Act: act, By: "operator", At: p.now(), Ask: w.Ask}
if o.ID != "yes" {
a.Said = "declined"
return p.finish(s, a)
}
var reasons []string
if act == ActTakeOver {
reasons = p.decide(w.Ask)
if s, err = p.readSwitch(); err != nil { // Decide clears a refusal and writes the switch: read it again
return err
}
} else if s.State == Shadow {
reasons = []string{"nothing to hand back: the bus node follows the controller's list already"}
}
if len(reasons) > 0 {
a.Said, a.Refused = "refused", strings.Join(reasons, "; ")
p.sayRefused(&s, act, a.Refused)
return p.finish(s, a)
}
p.clearRefused(&s)
a.Said = "recorded"
if act == ActTakeOver {
s.State = TookOver
} else {
s.State = Shadow
}
if err := p.finish(s, a); err != nil {
return err
}
p.recordWarrant(w.Ask)
// The bus node turns on the key: written now, so the switch reaches it on the operator's yes (ADR 0547 clause 8).
if err := p.syncFollow(s.State); err != nil {
return fmt.Errorf("the switch is recorded and the key the bus node follows could not be written (the next pass writes it): %w", err)
}
if p.Emit != nil {
_ = p.Emit.Emit(EventUserList, map[string]any{"switch": s.State})
}
return nil
}
// Said is the process's note of an ask that ended with no yes or no (expired, or refused by the router): the switch
// stays, and the ask is closed so the operator can be asked again.
func (p *Part) Said(h asker.Handled, w asks.Warrant, err error) {
if err != nil && p.Logf != nil {
p.Logf("the answer to %s (%s) ended in an error: %v", w.Ask, h, err)
}
if isLeaseAsk(w.Ask) {
if r, err := p.readLease(); err == nil && r.Open != nil && r.Open.Ask == w.Ask && r.Open.Step == StepAsked {
r.end(string(w.Outcome), string(w.Outcome)+" ("+string(h)+"): nothing was written", p.now())
_ = p.writeLease(r)
}
return
}
s, err := p.readSwitch()
if err != nil || s.Open != w.Ask {
return
}
_ = p.finish(s, Act{Act: actOf(w.Ask), By: "operator", At: p.now(), Ask: w.Ask, Said: string(w.Outcome) + " (" + string(h) + ")"})
}
func (p *Part) finish(s Switch, a Act) error {
if s.Open == a.Ask {
s.Open = ""
}
s.Acts = append(s.Acts, a)
return p.writeSwitch(s)
}
// Decide is what stands between a take-over and the grants part's list, read from a composition made now: the loaded list
// not read, any user the two lists keep apart other than by a right the shadow list adds (a user added, a loaded user
// dropped that placement still names, a right a loaded user would lose: ADR 0562, which changes ADR 0547 clause 8's
// equality), and any user with no hash in the vault.
// Nothing here but what is true now: a take-over is refused for these and for no other reason. Empty means none.
func (p *Part) Decide() []string {
return p.decide("")
}
// decide is Decide for the yes to the ask (none: empty). A right the shadow list adds is no reason (ADR 0562 §1): the ask
// names it, and the yes covers the rights the ask named and no other (ADR 0562 §3).
func (p *Part) decide(ask string) []string {
p.deciding.Store(true)
res := p.Compose()
p.deciding.Store(false)
reasons := Refusals(res.UserList.Compare, res.UserList.Credentials)
if ask != "" && p.askedAdds(ask) != res.UserList.Compare.AddedDigest() {
reasons = append(reasons, "the rights the grants part's list adds changed since you were asked, so the yes does not cover them: ask the take-over again")
}
reasons = append(reasons, p.followReasons(res, ReadInputs(p.Read, &p.cache))...)
if len(reasons) == 0 {
if s, err := p.readSwitch(); err == nil && len(s.Refusing) > 0 {
p.clearRefused(&s)
_ = p.writeSwitch(s)
}
}
return reasons
}
// askedAdds is the digest of the added rights the ask named: empty when its request is not the one kept.
func (p *Part) askedAdds(ask string) string {
raw, _, found, err := p.Store.Get(StateTakeover, KeyRequest)
if err != nil || !found {
return ""
}
var r Request
if json.Unmarshal(raw, &r) != nil || r.ID != ask {
return ""
}
return r.Adds
}
func contains(s []string, v string) bool {
for _, x := range s {
if x == v {
return true
}
}
return false
}
// sayRefused tells the operator a yes was given and the switch refused, with the reasons (the users that differ named). The
// condition stays raised until a switch is recorded or a later Decide finds no reason (clearRefused).
func (p *Part) sayRefused(s *Switch, act, why string) {
if p.Say == nil {
return
}
c := asks.Condition{Key: "mesh-grants." + act + "-refused", Severity: asks.SeverityWarning, Subject: asks.Scope{Scope: "module", ID: "mesh-grants"},
State: asks.StateRaised, Summary: act + " refused: " + why, Headline: "The user list switch was refused", Needs: "a look at why",
Explanation: asks.NeedsYou + "a look at why. You said yes to the switch and it was refused: " + why, Resolved: "the switch is asked again and goes through"}
if p.Say(c) == nil && !contains(s.Refusing, act) {
s.Refusing = append(s.Refusing, act)
}
}
// clearRefused clears every refusal condition said and not cleared: on a recorded switch, a hand-back, or a Decide with no reason.
func (p *Part) clearRefused(s *Switch) {
if p.Say == nil {
return
}
var left []string
for _, act := range s.Refusing {
c := asks.Condition{Key: "mesh-grants." + act + "-refused", Subject: asks.Scope{Scope: "module", ID: "mesh-grants"}, State: asks.StateCleared}
if p.Say(c) != nil {
left = append(left, act)
}
}
s.Refusing = left
}
// TakeOverState is the tool take-over-state: the switch, the request, and what became of each ask.
func (p *Part) TakeOverState() (any, error) {
s, err := p.readSwitch()
if err != nil {
return nil, err
}
loaded := ReadInputs(p.Read, &p.cache).Loaded
out := map[string]any{"state": s.State, "open": s.Open, "acts": s.Acts, "bus_node_loaded_grants_list": p.Following(),
"effect": p.Effect(s.State, loaded),
"says": "a recorded switch takes effect when the bus node loads the grants part's list; effect says switched only once the bus node's loaded digest is the grants part's written one"}
if g, rev, had := p.Sealed(); had {
out["written"] = map[string]any{"digest": g.Digest, "revision": g.Revision, "key_revision": rev, "follow": g.Follow,
"users": g.Users, "left_out": g.LeftOut, "sealed_to": g.Node}
}
if loaded != nil {
out["loaded"] = map[string]any{"source": loaded.Source, "digest": loaded.Digest, "revision": loaded.Revision,
"users": loaded.Users, "follows": loaded.Follows, "at": loaded.At, "error": loaded.Error}
}
if raw, _, found, err := p.Store.Get(StateTakeover, KeyRequest); err == nil && found {
out["request"] = json.RawMessage(raw)
}
return out, nil
}
// Module is the grants part's module, the asker of its take-over and hand-back.
// Module is the grants part's module.
const Module = "mesh-grants"
// The hand-act log's verb the part records its switches by (novox/hq ADR 0565 §11, ADR 0547 §6): the seat's.
const CallRecordWarrant = "seat:" + handacts.Seat + ".warranted"
// recordWarrant records, in the hand-act log, that a take-over or a hand-back was made on the operator's warrant for ask.
// Said, never fatal: the switch stands either way.
func (p *Part) recordWarrant(ask string) {
if p.Call == nil {
return
}
rec := handacts.Recorder{Ask: p.Call.Call}
taken, err := rec.RecordWarrant(Module, ask)
if p.Logf == nil {
return
}
if err != nil {
p.Logf("the switch on the warrant %s is not recorded in the hand-act log: %v", ask, err)
return
}
p.Logf("the switch on the warrant %s is recorded in the log of %s", ask, taken.Log)
}
// InForce is the mode of the part's user list: the bus node follows it alone (novox/hq ADR 0547 clause 8). The
// operator's take-over recorded it on 2026-10-27, and the controller's end by its unassignment made it the only list
// (ADR 0590 §3, issue 1126): no take-over and no hand-back are asked any more.
const InForce = "took-over"
modules/mesh-grants/internal/grants/tools.go
+12 -25
View File
@@ -7,33 +7,26 @@ import (
"slices"
"strings"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/membership"
)
// The part's tools, as its manifest lists them. Module tools in this milestone: a `mesh-*` seat must be in mesh-sdk's own
// seat list, which the running controller's composer does not know, so the part claims no seat until take-over.
// The part's tools, as its manifest lists them. The part claims no seat: the seat mesh-grants served the lease write
// alone, which went with the controller (novox/hq ADR 0590 §3, issue 1126).
const (
ToolGrants = "grants_grants"
ToolMembership = "grants_membership"
ToolUserList = "grants_user_list"
ToolComposeNow = "grants_compose_now"
ToolAssert = "grants_assert"
ToolTakeOver = "grants_take_over"
ToolTakeOverState = "grants_take_over_state"
ToolHandBack = "grants_hand_back"
ToolAccounts = "grants_accounts"
ToolConsumerMatch = "grants_consumer_match"
ToolAcceptHashes = "grants_accept_hashes"
)
// Tools is every tool, in the manifest's order.
var Tools = []string{ToolGrants, ToolMembership, ToolUserList, ToolComposeNow, ToolAssert, ToolTakeOver, ToolHandBack, ToolTakeOverState,
ToolAccounts, ToolConsumerMatch, ToolAcceptHashes, ToolLeaseCheck, ToolRemovalDue}
// SeatTool is the tool serving one of the seat mesh-grants' verbs, `mesh-grants.<verb>`.
func SeatTool(verb string) string { return catalogue.GrantsSeat + "." + verb }
var Tools = []string{ToolGrants, ToolMembership, ToolUserList, ToolComposeNow, ToolAssert, ToolAccounts, ToolConsumerMatch,
ToolAcceptHashes}
// WritesGrants is what grants says of the part's own grants: written once the list the bus loaded carries a build's
// accounts, and the one key the assigner reads (ADR 0571 clause 9, which changes ADR 0547 clause 5).
@@ -42,14 +35,14 @@ const WritesGrants = "the grants part writes grants.<module>.<fp> into its state
// Grants answers the tool grants: the grants the part wrote, one module's or all.
func (p *Part) Grants(module string) (any, error) {
out := map[string]any{"writes": WritesGrants}
read := func(get func() ([]string, func(string) ([]byte, bool, error), error)) (map[string]delivery.ControllerGrant, error) {
read := func(get func() ([]string, func(string) ([]byte, bool, error), error)) (map[string]delivery.Grant, error) {
keys, one, err := get()
if err != nil {
return nil, err
}
m := map[string]delivery.ControllerGrant{}
m := map[string]delivery.Grant{}
for _, k := range keys {
mod, _, err := delivery.ParseControllerGrantKey(k)
mod, _, err := delivery.ParsePartGrantKey(k)
if err != nil || (module != "" && mod != module) {
continue
}
@@ -60,7 +53,7 @@ func (p *Part) Grants(module string) (any, error) {
if !found {
continue
}
g, err := delivery.DecodeControllerGrant(raw)
g, err := delivery.DecodeGrant(raw)
if err != nil {
return nil, fmt.Errorf("%s: %w", k, err)
}
@@ -80,10 +73,10 @@ func (p *Part) Grants(module string) (any, error) {
Module string `json:"module"`
Accounts []string `json:"accounts,omitempty"`
}
list := func(m map[string]delivery.ControllerGrant) []grant {
list := func(m map[string]delivery.Grant) []grant {
out := []grant{}
for _, k := range sortedKeys(m) {
mod, _, _ := delivery.ParseControllerGrantKey(k)
mod, _, _ := delivery.ParsePartGrantKey(k)
out = append(out, grant{Key: k, Module: mod, Accounts: m[k].Accounts})
}
return out
@@ -210,11 +203,6 @@ func (p *Part) Pass() Result {
return res
}
// NoGrantAgain is what compose-now says of the controller's grants: the controller serves no verb that writes a module's
// grant again (its issue replaces the account, which is not that), so a grant write it lost stays lost until it composes
// again; the part writes grants only from ADR 0547's third milestone.
const NoGrantAgain = "missing: a verb to ask the controller to write a module's grant again (mesh-controller serves none; its issue replaces the bus account, which is not that), so compose-now cannot ask it"
// AskCompose is the tool compose-now: it asks the part's process to compose and publish now (only the process holds the
// grant to publish a membership), and answers the last pass, whose revision moves when the process answered.
func (p *Part) AskCompose(caller string) (any, error) {
@@ -223,8 +211,7 @@ func (p *Part) AskCompose(caller string) (any, error) {
if err != nil {
return nil, fmt.Errorf("the ask could not be written: %w", err)
}
out := map[string]any{"asked": rev, "says": "the part's process composes and publishes at this ask; grants_user_list's last pass says what it published",
"controller_grants": NoGrantAgain}
out := map[string]any{"asked": rev, "says": "the part's process composes and publishes at this ask; grants_user_list's last pass says what it published"}
if raw, prev, found, err := p.Store.Get(StateUserList, KeyPass); err == nil && found {
out["last_pass"] = map[string]any{"revision": prev, "result": json.RawMessage(raw)}
}
@@ -233,7 +220,7 @@ func (p *Part) AskCompose(caller string) (any, error) {
// UserList answers the tool user-list: the shadow list as last written, beside what the bus node says it loaded now.
func (p *Part) UserList() (any, error) {
out := map[string]any{"mode": Shadow}
out := map[string]any{"mode": InForce}
raw, rev, found, err := p.Store.Get(StateUserList, KeyList)
switch {
case err != nil:
modules/mesh-grants/internal/grants/userlist.go
+1 -220
View File
@@ -13,7 +13,6 @@ import (
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/outward"
)
// User is one bus user and its grants, as the bus module's nats_users answers them: never a password hash.
@@ -317,7 +316,7 @@ func (c Comparison) Removing() []string {
// AddedRights are the rights the shadow list adds, by user: every user's added publish and subscribe rights and an
// allowance of responses gained, whatever else the comparison holds. They are the whole difference only where Removing is
// empty, and only then is a take-over allowed for them (ADR 0562 §1, §2).
// empty (ADR 0562 §1, §2).
func (c Comparison) AddedRights() []UserDiff {
var out []UserDiff
for _, d := range c.Differ {
@@ -332,224 +331,6 @@ func (c Comparison) AddedRights() []UserDiff {
return out
}
// AddedDigest is the digest of every right the shadow list adds: what an ask names and its yes answers (ADR 0562 §3).
func (c Comparison) AddedDigest() string {
var lines []string
for _, d := range c.AddedRights() {
for _, s := range d.PublishAdded {
lines = append(lines, d.User+"\tpublish\t"+s)
}
for _, s := range d.SubscribeAdded {
lines = append(lines, d.User+"\tsubscribe\t"+s)
}
if d.Responses != "" {
lines = append(lines, d.User+"\tresponses")
}
}
sort.Strings(lines)
sum := sha256.Sum256([]byte(strings.Join(lines, "\n")))
return "sha256:" + hex.EncodeToString(sum[:])
}
// The kinds an added right is told by in the operator's ask.
const (
kindAdvisory = "advisories"
kindRead = "reads"
kindCall = "calls"
kindEvent = "events"
kindOther = "other"
)
var rightKinds = []string{kindRead, kindAdvisory, kindCall, kindEvent, kindOther}
func kindOfRight(subject string) string {
switch {
case strings.HasPrefix(subject, "$JS.EVENT.ADVISORY."):
return kindAdvisory
case strings.HasPrefix(subject, "$JS.") || strings.HasPrefix(subject, "$KV.") || strings.HasPrefix(subject, "$O."):
return kindRead
case strings.HasPrefix(subject, "mesh.seat."):
return kindCall
case strings.HasPrefix(subject, "mesh.mod.") && strings.Contains(subject, ".event."):
return kindEvent
}
return kindOther
}
// kindWords says what each kind is, in the operator's words.
var kindWords = map[string]string{kindRead: "reads of buckets and streams", kindAdvisory: "stream-created advisories",
kindCall: "seat calls", kindEvent: "events", kindOther: "other subjects"}
// tally is one user's added rights counted by kind, as "36 reads, 1 call"; responses allowed are said too.
func tally(d UserDiff) string {
n := map[string]int{}
for _, s := range append(append([]string(nil), d.PublishAdded...), d.SubscribeAdded...) {
n[kindOfRight(s)]++
}
var parts []string
for _, k := range rightKinds {
if n[k] > 0 {
word := k
if n[k] == 1 {
word = map[string]string{kindAdvisory: "advisory", kindRead: "read", kindCall: "call", kindEvent: "event", kindOther: "other"}[k]
}
parts = append(parts, fmt.Sprintf("%d %s", n[k], word))
}
}
if d.Responses != "" {
parts = append(parts, "responses allowed")
}
return strings.Join(parts, ", ")
}
// AddedWordsBudget is how many characters the ask's list of added rights may take, so the ask fits one message on a
// channel such as Telegram (4096 characters) with its headline, explanation and options.
const AddedWordsBudget = 2200
// AddedWords are an ask's words for the rights the shadow list adds (ADR 0562 §3): a summary, a point per kind with its
// total, and the list by user, users with the same count grouped on one line; within the content rule (outward.Check,
// a name that would not pass written with spaces) and AddedWordsBudget, past which the last lines are counted, not named.
func (c Comparison) AddedWords() (summary string, points []string, list string) {
adds := c.AddedRights()
if len(adds) == 0 || c.Unread != "" || len(c.Removing()) > 0 {
return "", nil, ""
}
total, byKind, usersOf := 0, map[string]int{}, map[string]int{}
group := map[string][]string{}
for _, d := range adds {
seen := map[string]bool{}
for _, s := range append(append([]string(nil), d.PublishAdded...), d.SubscribeAdded...) {
k := kindOfRight(s)
total++
byKind[k]++
if !seen[k] {
seen[k] = true
usersOf[k]++
}
}
t := tally(d)
group[t] = append(group[t], plainName(d.User))
}
summary = fmt.Sprintf("The grants part's list adds %s to %s and takes none away; no user, hash or login changes.", count(total, "right"), count(len(adds), "user"))
var kinds []string
for _, k := range rightKinds {
if byKind[k] > 0 {
kinds = append(kinds, fmt.Sprintf("%d %s (%s)", byKind[k], kindWords[k], count(usersOf[k], "user")))
}
}
// One point of the kinds, wrapped at the explanation's point length.
line := "Added: "
for i, k := range kinds {
if i > 0 && len([]rune(line))+len([]rune(k))+2 > 160 {
points = append(points, strings.TrimSuffix(line, ", "))
line = "Added: "
}
line += k + ", "
}
points = append(points, strings.TrimSuffix(line, ", ")+".")
if n := len(c.Excused); n > 0 {
points = append(points, fmt.Sprintf("Not counted: %d rights excused as before (a retired verb, a seat held on several machines).", n))
}
tallies := sortedKeys(group)
sort.SliceStable(tallies, func(i, j int) bool { return len(group[tallies[i]]) > len(group[tallies[j]]) })
var lines []string
used := 0
for i, t := range tallies {
users := group[t]
sort.Strings(users)
l := strings.Join(users, ", ") + ": " + t
if len(users) > 1 {
l += " each"
}
if used+len([]rune(l))+1 > AddedWordsBudget-80 {
rest := 0
for _, more := range tallies[i:] {
rest += len(group[more])
}
lines = append(lines, fmt.Sprintf("and %d more users: grants_user_list names every right added.", rest))
break
}
lines = append(lines, l)
used += len([]rune(l)) + 1
}
return summary, points, strings.Join(lines, "\n")
}
// count is n things, with the plural where n is not one.
func count(n int, thing string) string {
if n == 1 {
return "1 " + thing
}
return fmt.Sprintf("%d %ss", n, thing)
}
// Refusals are why a take-over is refused for the lists themselves, each naming its users: the loaded list unread, any user
// the lists keep apart other than by an added right (Removing), and any user with no hash in the vault but those both
// lists agree are gone. Empty when the lists differ at most by added rights and every login is kept.
func Refusals(c Comparison, cr Credentials) []string {
var reasons []string
if c.Unread != "" {
reasons = append(reasons, "the loaded list could not be read: "+c.Unread)
}
if d := c.Removing(); len(d) > 0 {
reasons = append(reasons, fmt.Sprintf("%d users differ between the shadow list and the loaded list other than by an added right: %s", len(d), strings.Join(d, ", ")))
}
var unhashed []string
for _, u := range append(append([]string(nil), cr.LeftOut...), cr.LosesLogin...) {
if !slices.Contains(c.Gone, u) {
unhashed = append(unhashed, u)
}
}
if len(unhashed) > 0 {
sort.Strings(unhashed)
reasons = append(reasons, fmt.Sprintf("the vault holds no hash for %d users (ADR 0547 clause 9: handed or adopted before take-over): %s",
len(unhashed), strings.Join(unhashed, ", ")))
}
return reasons
}
// TakeOverWords are the take-over ask's words for the lists as compared now: the rights the grants part's list adds
// (AddedWords) when nothing else keeps them apart, or else what a yes would be refused for, the users named in its list
// (ADR 0562 §2, §3). Empty when the lists agree. Held to the content rule: the loaded list's read error is not said.
func TakeOverWords(c Comparison, cr Credentials) (summary string, points []string, list string) {
refusals := Refusals(c, cr)
if len(refusals) == 0 {
return c.AddedWords()
}
summary = "Today a yes would be refused: the lists differ other than by an added right, the loaded list is unread, or a user has no hash."
var lines []string
if c.Unread != "" {
lines = append(lines, "The loaded list could not be read.")
refusals = refusals[1:]
}
used := 0
for _, r := range refusals {
if used+len([]rune(r)) > AddedWordsBudget-80 {
r = string([]rune(r)[:max(0, AddedWordsBudget-80-used)]) + "… grants_take_over_state says the rest."
}
lines = append(lines, r)
used += len([]rune(r)) + 1
if used >= AddedWordsBudget-80 {
break
}
}
for i := range lines {
if _, ok := outward.Check(lines[i]); !ok {
lines[i] = strings.ReplaceAll(lines[i], ".", " ")
}
}
return summary, []string{"Your yes now is refused and said, naming them; ask again once they agree."}, strings.Join(lines, "\n")
}
// plainName is a user's name as the operator channel's content rule lets it pass: as it is, or with spaces for its dots
// where it reads as a host name.
func plainName(user string) string {
if _, ok := outward.Check(user); ok {
return user
}
return strings.ReplaceAll(user, ".", " ")
}
// Compare compares the shadow list with the loaded one. placed says whether placement still names a loaded user's
// machine and module; nil names every one. Pure.
func Compare(shadow, loaded []User, placed func(user string) bool) Comparison {
modules/mesh-grants/leasecheck_test.go
-121
View File
@@ -1,121 +0,0 @@
package manifest
import (
"encoding/json"
"slices"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-grants/internal/grants"
)
// Issue 774, design 47 L3 (ADR 0547 §11, ADR 0561 §3, ADR 0571 §13): grants_lease_check answers from bus state who holds
// the controller's lease, every manifest's dependency on the controller with what keeps it, the controller's streams in
// use, and the lease write's check run dry, at the write (the three kept to it alone) and before its delivery. The
// judgement of each kind of dependency is mesh-sdk's and tested there (catalogue/controller_dependence_test.go, its rows
// moved from here, 774/S4); these rows are the tool's own: what it read, the lease, the streams.
func TestTheLeaseCheckJudgesEveryDependencyOnTheController(t *testing.T) {
const mManifest = `{"module":"mod-m","version":"1","reads":["mesh-controller.declared-packages"]}`
controllerHolds := map[string]any{"instance": "controller@novox-be pid 1 since 2026-10-30T14:52:10Z", "epoch": 60,
"taken": "2026-10-30T14:52:10Z", "renewed": "2026-10-30T22:00:00Z"}
handedAway := map[string]any{"handed-to": "the operator's hand-over", "handed-by": "operator"}
type want struct {
deps []string // "<module> <kind> <what> <kept?>" with kept as "kept" or "unkept"
writePasses bool
beforePasses bool
reason string // a reason the check at the write must name, or ""
bothPhases bool // the check before the delivery must name the reason too
handedAway bool
durable []string // durable consumers of KV_mesh-controller_assignments
}
for _, row := range []struct {
name string
manifests map[string]string
lease map[string]any
want want
setup func(b *fakeBus)
}{
{"a read of the lease alone is no dependency, and the controller's own manifest is not judged",
map[string]string{"mod-l": `{"module":"mod-l","version":"1","reads":["mesh-controller.lease"]}`,
"mesh-controller": `{"module":"mesh-controller","version":"1","invokes":["seat:mesh-controller.status"]}`},
controllerHolds, want{writePasses: true, beforePasses: true}, nil},
{"no manifest read from the indexer fails both phases, though nothing read depends on the controller",
map[string]string{},
controllerHolds, want{reason: "no manifest was read from the indexer (it holds 0 modules)", bothPhases: true}, nil},
{"a module whose manifest digest is missing fails both phases, named",
map[string]string{"mod-l": `{"module":"mod-l","version":"1"}`, "mod-m": mManifest},
controllerHolds, want{reason: "manifest of mod-m could not be read", bothPhases: true}, func(b *fakeBus) {
delete(b.states[grants.ReadManifests], delivery.ManifestDigest([]byte(mManifest)))
}},
{"a manifest the indexer's judge refuses fails both phases, named",
map[string]string{"mod-l": `{"module":"mod-l","version":"1"}`, "mod-r": `{"module":"mod-r","version":"1","waits":["mesh-controller.checked"]}`},
controllerHolds, want{reason: "manifest of mod-r could not be read", bothPhases: true}, func(b *fakeBus) {
b.refuse("mod-r", `{"module":"mod-r","version":"1","waits":["mesh-controller.checked"]}`, "waits on what no record keeps")
}},
{"a lease already handed away leaves nothing to write",
map[string]string{},
handedAway, want{handedAway: true, reason: "there is nothing to write: the lease is handed to the operator's hand-over"}, nil},
} {
t.Run(row.name, func(t *testing.T) {
b := newFakeBus()
for m, raw := range row.manifests {
b.manifestOf(m, raw)
}
b.put(grants.ReadLease, grants.LeaseKey, row.lease)
if row.setup != nil {
row.setup(b)
}
b.streams = []map[string]any{
{"name": "KV_mesh-controller_assignments", "consumers": 3, "messages": 2991, "subjects_held": 40, "last": "2026-10-30T12:55:13Z"},
{"name": "KV_mesh-controller_rollouts", "consumers": 0, "messages": 6652},
{"name": "KV_mesh-grants_memberships", "consumers": 243, "messages": 1573},
}
b.consumers = map[string][]map[string]any{"KV_mesh-controller_assignments": {{"name": "0AYfjxta"}, {"name": "1g9XJFUX"}, {"name": "grants-assignments"}}}
raw, err := partOn(b).LeaseCheck()
if err != nil {
t.Fatal(err)
}
var got grants.LeaseAnswer
data, _ := json.Marshal(raw)
if err := json.Unmarshal(data, &got); err != nil {
t.Fatal(err)
}
var deps []string
for _, d := range got.Dependencies {
kept := "unkept"
if d.Kept != "" {
kept = "kept"
}
deps = append(deps, d.Module+" "+d.Kind+" "+d.What+" "+kept)
}
if !slices.Equal(deps, row.want.deps) {
t.Errorf("dependencies %q, want %q", deps, row.want.deps)
}
if got.Check.Phase != grants.PhaseWrite || got.Check.Passes != row.want.writePasses {
t.Errorf("check at the write passes=%v (%v), want %v", got.Check.Passes, got.Check.Reasons, row.want.writePasses)
}
if got.Before.Phase != grants.PhaseBefore || got.Before.Passes != row.want.beforePasses {
t.Errorf("check before the delivery passes=%v (%v), want %v", got.Before.Passes, got.Before.Reasons, row.want.beforePasses)
}
if row.want.reason != "" && !slices.ContainsFunc(got.Check.Reasons, func(r string) bool { return strings.Contains(r, row.want.reason) }) {
t.Errorf("reasons %q name nothing like %q", got.Check.Reasons, row.want.reason)
}
if row.want.bothPhases && !slices.ContainsFunc(got.Before.Reasons, func(r string) bool { return strings.Contains(r, row.want.reason) }) {
t.Errorf("reasons before the delivery %q name nothing like %q", got.Before.Reasons, row.want.reason)
}
if got.Lease.HandedAway != row.want.handedAway {
t.Errorf("lease handed away %v (%s), want %v", got.Lease.HandedAway, got.Lease.Says, row.want.handedAway)
}
if !row.want.handedAway && (got.Lease.Since == nil || got.Lease.Since.Format("15:04") != "14:52" ||
!strings.Contains(got.Lease.Says, "the controller holds the lease") || !slices.Equal(got.Lease.Does, grants.DoesWhileController)) {
t.Errorf("lease %+v: the controller's instance since 14:52, and what it does while it holds it", got.Lease)
}
if len(got.Streams) != 1 || got.Streams[0].Stream != "KV_mesh-controller_assignments" || got.Streams[0].Consumers != 3 ||
!slices.Equal(got.Streams[0].Durable, []string{"grants-assignments"}) || !slices.Equal(got.Idle, []string{"KV_mesh-controller_rollouts"}) {
t.Errorf("streams %+v idle %v: the controller's assignments bucket with its one durable consumer, the rollouts idle, no other module's", got.Streams, got.Idle)
}
})
}
}
modules/mesh-grants/leasewrite_test.go
-503
View File
@@ -1,503 +0,0 @@
package manifest
import (
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-sdk/go/seal"
"mesh-grants/internal/grants"
)
// leaseFake is the bus as the lease write meets it: fakeBus's states and tools, and the seats and tools the write asks.
// Each answer is shaped as its writer's code answers it (issue 774): node-service-manager's status the systemd module's
// Manager.Status (ActiveState), nats_bucket_expiry nats-tools' BucketExpirySet (now), nats_consumer_remove its
// ConsumerRemoved (removed) and an absent consumer its error ("has no consumer"), nats_stream_remove its StreamRemoved
// (removed, present), issue-tracker.get mesh-issues' Brief (status), node_trusts mesh-delivery-view's TrustsAnswer
// (every), mesh-config's compare its Comparison (source, count), mesh-wants' upgrade compare the assigner's answer
// (empty), vault_status mesh-vault's answer (ready, minting.on, minting.held_back).
type leaseFake struct {
*fakeBus
answer map[string]func(map[string]any) (any, error)
active string // the controller's ActiveState
expiry time.Duration // the lease bucket's
ttls map[string]time.Duration
log []string // the write's acts on the bus, in order
logged []string // the process's log lines
}
func newLeaseFake(t *testing.T) (*leaseFake, *grants.Part) {
t.Helper()
b := fixture()
b.manifestOf("nats", `{"module":"nats","version":"1","claims":[{"name":"mesh-broker"}],"own-secrets":{"broker":"/var/lib/nats/broker"},`+
`"tools":["nats_bucket_expiry","nats_consumer_remove","nats_stream_remove"]}`)
// The lease's host is the machine's hostname, never a node's name (issue 774: "novox-be" on the node novox); the
// controller's node is the one placement places it on.
b.put(grants.ReadLease, grants.LeaseKey, map[string]any{"instance": "controller@box-a pid 1", "host": "box-a", "epoch": 60,
"taken": "2026-10-30T14:52:10Z"})
b.assign("a", grants.Controller)
b.put(grants.ReadTakeover, health.TakeoverKey, health.Takeover{V: 1, Follow: health.FollowKeeper, By: "operator", Why: "the keeper took over"})
busKey, err := seal.GenerateKeyPair()
if err != nil {
t.Fatal(err)
}
b.put(grants.ReadHave, "a.nats", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK,
UserList: &delivery.UserListWritten{Source: "a.nats.src2", Revision: 7, Digest: "sha256:aa", Users: 6, Key: busKey.PublicKey}})
// The controller's hash in the vault, so the list carries the controller and the cut changes it.
b.put(grants.ReadAccounts, buscompose.ControllerName, map[string]any{"hash": "$2a$11$controller", "generation": 1,
"at": "2026-10-30T14:52:10Z"})
f := &leaseFake{fakeBus: b, active: "active", expiry: 15 * time.Second, ttls: map[string]time.Duration{}}
f.answer = map[string]func(map[string]any) (any, error){
grants.CallIssue: func(map[string]any) (any, error) { return map[string]any{"status": "resolved"}, nil },
grants.CallNodeTrusts: func(map[string]any) (any, error) { return map[string]any{"every": true, "not_yet": []string{}}, nil },
grants.CallConfigCompare: func(map[string]any) (any, error) { return map[string]any{"source": "indexer", "count": 0}, nil },
grants.CallUpgrade: func(map[string]any) (any, error) { return map[string]any{"empty": true, "splits": []any{}}, nil },
grants.CallVaultStatus: func(map[string]any) (any, error) {
return map[string]any{"ready": true, "minting": map[string]any{"on": true, "held_back": map[string]string{}}}, nil
},
grants.CallServiceStop: func(map[string]any) (any, error) { f.active = "inactive"; return map[string]any{"ok": true}, nil },
grants.CallServiceStart: func(map[string]any) (any, error) { f.active = "active"; return map[string]any{"ok": true}, nil },
grants.CallServiceStatus: func(map[string]any) (any, error) { return map[string]any{"ActiveState": f.active}, nil },
grants.CallBucketExpiry: func(a map[string]any) (any, error) {
now := fmt.Sprint(a["max_age"])
d, _ := time.ParseDuration(strings.Replace(now, "none", "0s", 1))
f.expiry = d
f.log = append(f.log, "expiry "+now)
return map[string]any{"bucket": a["bucket"], "now": now, "changed": true}, nil
},
grants.CallRecordWarrant: func(a map[string]any) (any, error) { f.log = append(f.log, "warranted"); return map[string]any{}, nil },
grants.CallRecordHandAct: func(a map[string]any) (any, error) {
f.log = append(f.log, "hand-act "+fmt.Sprint(a["what"]))
return map[string]any{}, nil
},
grants.CallConsumerRemove: func(map[string]any) (any, error) {
f.log = append(f.log, "consumer removed")
return map[string]any{"removed": true}, nil
},
grants.CallStreamRemove: func(map[string]any) (any, error) {
f.log = append(f.log, "stream removed")
return map[string]any{"removed": true, "present": true}, nil
},
}
p := partOn(b)
p.Call, p.Store, p.Lease, p.Asker, p.Objects = f, ttlStore{ownStore{b}, f}, f, &fakeAsker{wheres: map[string]*asks.Warrant{}}, &fakeObjects{}
p.SaySeat = func(event string, _ any) error { f.log = append(f.log, "event "+event); return nil }
p.Nonce = func() string { return "0001" }
p.Logf = func(format string, a ...any) { f.logged = append(f.logged, fmt.Sprintf(format, a...)) }
return f, p
}
func (f *leaseFake) Call(key string, body any) (json.RawMessage, error) {
base, _, _ := strings.Cut(key, "@")
if answer, ok := f.answer[base]; ok {
f.mu.Lock()
f.calls = append(f.calls, key)
f.mu.Unlock()
args, _ := body.(map[string]any)
out, err := answer(args)
if err != nil {
return nil, err
}
return json.Marshal(out)
}
return f.fakeBus.Call(key, body)
}
func (f *leaseFake) PutHolder(value []byte) error {
f.log = append(f.log, "holder")
f.states[grants.ReadLease][grants.LeaseKey] = value
return nil
}
func (f *leaseFake) DeleteHolder() error {
f.log = append(f.log, "holder deleted")
delete(f.states[grants.ReadLease], grants.LeaseKey)
return nil
}
func (f *leaseFake) Expiry() (time.Duration, error) { return f.expiry, nil }
// loads says the bus node loaded the list the part last sealed.
func (f *leaseFake) loads(p *grants.Part) {
g, _, _ := p.Sealed()
f.put(grants.ReadHave, "a.nats", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK,
UserList: &delivery.UserListWritten{Source: delivery.GrantsUserListSource, Revision: g.Revision, Digest: g.Digest,
Users: g.Users, Key: keyOf(f)}})
}
func keyOf(f *leaseFake) string {
var h delivery.Have
_ = json.Unmarshal(f.states[grants.ReadHave]["a.nats"], &h)
return h.UserList.Key
}
// ttlStore is the part's own store with a time to live per key, recorded.
type ttlStore struct {
ownStore
f *leaseFake
}
func (s ttlStore) PutTTL(state, key string, value []byte, ttl time.Duration) (uint64, error) {
s.f.ttls[state+"."+key] = ttl
s.f.log = append(s.f.log, key+" "+ttl.String())
return s.Put(state, key, value)
}
// record is the part's lease record now.
func record(t *testing.T, p *grants.Part) grants.LeaseRecord {
t.Helper()
r, err := p.LeaseState()
if err != nil {
t.Fatal(err)
}
return r
}
// warrant answers the open act's ask with option, as the router's warrant hands it.
func warrant(t *testing.T, p *grants.Part, option string) error {
t.Helper()
r := record(t, p)
if r.Open == nil {
t.Fatal("no act is open")
}
return p.Warranted(asks.Option{ID: option}, asks.Warrant{Ask: r.Open.Ask, Option: option, Outcome: asks.OutcomeChosen})
}
// Issue 774 (novox/hq ADR 0582): the lease write, from its ask through the day-later removal, against stand-in seats.
func TestTheLeaseWrite(t *testing.T) {
// 774/S14/AC1: any one precondition not green refuses the verb naming it; all green asks.
t.Run("preconditions", func(t *testing.T) {
for _, row := range []struct {
name string
break_ func(f *leaseFake)
}{
{"lease-check", func(f *leaseFake) {
f.manifestOf("mod-u", `{"module":"mod-u","version":"1","reads":["mesh-controller.conditions"]}`)
}},
{"earlier-steps", func(f *leaseFake) {
f.answer[grants.CallIssue] = func(a map[string]any) (any, error) {
if fmt.Sprint(a["number"]) == "1954" {
return map[string]any{"status": "fixing"}, nil
}
return map[string]any{"status": "resolved"}, nil
}
}},
{"node-trusts", func(f *leaseFake) {
f.answer[grants.CallNodeTrusts] = func(map[string]any) (any, error) {
return map[string]any{"every": false, "not_yet": []string{"ace"}}, nil
}
}},
{"config-compare", func(f *leaseFake) {
f.answer[grants.CallConfigCompare] = func(map[string]any) (any, error) { return map[string]any{"source": "indexer", "count": 2}, nil }
}},
{"upgrade-compare", func(f *leaseFake) {
f.answer[grants.CallUpgrade] = func(map[string]any) (any, error) { return map[string]any{"empty": false, "splits": []any{1}}, nil }
}},
{"keeper-take-over", func(f *leaseFake) {
f.put(grants.ReadTakeover, health.TakeoverKey, health.Takeover{V: 1, Follow: health.FollowController})
}},
{"vault-minting", func(f *leaseFake) {
f.answer[grants.CallVaultStatus] = func(map[string]any) (any, error) {
return map[string]any{"ready": true, "minting": map[string]any{"on": true, "held_back": map[string]string{"a.x": "why"}}}, nil
}
}},
{"bus-tools", func(f *leaseFake) { f.manifestOf("nats", natsManifest) }},
{"", func(*leaseFake) {}},
} {
f, p := newLeaseFake(t)
row.break_(f)
_, err := p.LeaseWrite("operator@shanks", "the controller goes")
switch {
case row.name == "" && err != nil:
t.Errorf("all green: refused: %v", err)
case row.name != "" && (err == nil || !strings.Contains(err.Error(), row.name+": ")):
t.Errorf("%s not green: %v, want a refusal naming it", row.name, err)
case row.name != "":
for _, other := range grants.LeaseCheckNames {
if other != row.name && strings.Contains(err.Error(), other+": ") {
t.Errorf("%s not green: the refusal names %s too: %v", row.name, other, err)
}
}
}
if r := record(t, p); (r.Open != nil) != (row.name == "") {
t.Errorf("%q: an act open %v", row.name, r.Open)
}
}
})
// 774/S15/AC1, AC2: a no, an expiry, or a warrant whose checks changed writes nothing, naming the changed check.
t.Run("the ask and the warrant", func(t *testing.T) {
for _, row := range []struct {
name, step, said string
answer func(t *testing.T, f *leaseFake, p *grants.Part)
}{
{"declined", grants.StepDeclined, "said no", func(t *testing.T, _ *leaseFake, p *grants.Part) {
if err := warrant(t, p, "no"); err != nil {
t.Fatal(err)
}
}},
{"expired", "expired", "expired", func(t *testing.T, _ *leaseFake, p *grants.Part) {
r := record(t, p)
p.Said("", asks.Warrant{Ask: r.Open.Ask, Outcome: asks.OutcomeExpired}, nil)
}},
{"checks changed", grants.StepRefused, "the checks changed since the ask: vault-minting (now: ready true, minting true, 1 held back)",
func(t *testing.T, f *leaseFake, p *grants.Part) {
// The vault holds one account back between the ask and the warrant.
f.answer[grants.CallVaultStatus] = func(map[string]any) (any, error) {
return map[string]any{"ready": true, "minting": map[string]any{"on": true, "held_back": map[string]string{"a.x": "why"}}}, nil
}
if err := warrant(t, p, "yes"); err != nil {
t.Fatal(err)
}
}},
} {
f, p := newLeaseFake(t)
if _, err := p.LeaseWrite("operator@shanks", "the controller goes"); err != nil {
t.Fatal(err)
}
p.Compose() // the process asks it
if a := p.Asker.(*fakeAsker); len(a.asks) != 1 || len(a.asks[0].Options) != 2 || a.asks[0].Options[0].Binds == "" {
t.Fatalf("%s: asked %+v, want one ask whose yes binds the act", row.name, a.asks)
}
row.answer(t, f, p)
r := record(t, p)
if r.Open != nil || len(r.Acts) != 1 || r.Acts[0].Step != row.step || !strings.Contains(r.Acts[0].Said, row.said) {
t.Errorf("%s: open %v, acts %+v; want one act %s saying %q", row.name, r.Open, r.Acts, row.step, row.said)
}
if f.count(grants.CallServiceStop+"@a") != 0 || len(f.log) != 0 || r.Cut {
t.Errorf("%s: something was written: stops %d, %v, cut %v", row.name, f.count(grants.CallServiceStop+"@a"), f.log, r.Cut)
}
}
})
// 774/S16/AC1, AC2; 774/S17/AC1, AC2: the write's six steps, in order, each read back before the next.
t.Run("the write", func(t *testing.T) {
f, p := newLeaseFake(t)
f.answer[grants.CallServiceStop] = func(map[string]any) (any, error) { return map[string]any{"ok": true}, nil } // reads back active
if _, err := p.LeaseWrite("operator@shanks", "the controller goes"); err != nil {
t.Fatal(err)
}
p.Compose()
if err := warrant(t, p, "yes"); err != nil {
t.Fatal(err)
}
if r := record(t, p); r.Open != nil || r.Cut || r.Acts[0].Step != grants.StepFailed || !strings.Contains(r.Acts[0].Said, "did not read back stopped") || len(f.log) != 0 {
t.Fatalf("a stop not read back: %+v, %v; want the act failed there, nothing written", r, f.log)
}
f, p = newLeaseFake(t)
if _, err := p.LeaseWrite("operator@shanks", "the controller goes"); err != nil {
t.Fatal(err)
}
p.Compose()
before, _, _ := p.Sealed()
if err := warrant(t, p, "yes"); err != nil {
t.Fatal(err)
}
r := record(t, p)
g, _, _ := p.Sealed()
if f.active != "inactive" || !r.Cut || r.Open == nil || r.Open.Step != grants.StepCut || r.Open.List != g.Digest || g.Digest == before.Digest {
t.Fatalf("after the warrant: active %s, %+v, sealed %s (was %s); want stopped, cut and waiting on the cut list", f.active, r, g.Digest, before.Digest)
}
if len(f.log) != 0 {
t.Fatalf("the expiry or the key was touched before the cut list loaded: %v", f.log)
}
// The cut list takes the controller's write on the lease and gives mesh-grants the key holder.
var st grants.UserListState
raw, _, _, _ := ownStore{f.fakeBus}.Get(grants.StateUserList, grants.KeyList)
_ = json.Unmarshal(raw, &st)
p.Compose() // a pass with the bus node not loaded yet waits
if len(f.log) != 0 {
t.Fatalf("a pass before the load acted: %v", f.log)
}
f.loads(p)
p.Compose()
want := []string{"expiry none", "holder", "event lease-handed", "warranted", "removal-due 24h0m0s"}
if !slices.Equal(f.log, want) {
t.Errorf("the write's acts %v, want %v in that order", f.log, want)
}
var h buscompose.LeaseHolder
_ = json.Unmarshal(f.states[grants.ReadLease][grants.LeaseKey], &h)
r = record(t, p)
if h.HandedTo != "mesh-grants" || h.HandedBy != "operator" || h.Why != "the controller goes" || h.Ask != r.Ask || h.At.IsZero() ||
r.Open != nil || r.Handed.IsZero() || r.Acts[len(r.Acts)-1].Step != grants.StepDone {
t.Errorf("holder %+v, record %+v", h, r)
}
p.Compose() // the pass the lease key's change wakes
if g, _, _ := p.Sealed(); !g.LeaseHanded {
t.Error("the user-list key does not say the lease is handed")
}
if _, err := p.HandBack("operator@shanks", "back"); err == nil || !strings.HasPrefix(err.Error(), "gone") {
t.Errorf("the user list's hand-back once the lease is handed: %v, want gone", err)
}
if _, err := p.LeaseWrite("operator@shanks", "again"); err == nil {
t.Error("a second lease-write with the lease handed was asked")
}
// 774/S18/AC1: the key or the expiry changed by another hand is said once, and nothing is written.
f.log = nil
f.put(grants.ReadLease, grants.LeaseKey, map[string]any{"instance": "controller@a pid 9", "host": "box-a", "epoch": 61})
p.Compose()
p.Compose()
f.expiry = 15 * time.Second
p.Compose()
if !slices.Equal(f.log, []string{"event lease-altered"}) || !strings.Contains(record(t, p).Altered, "controller@a pid 9") {
t.Errorf("an altered lease said %v (%q), want lease-altered once naming it", f.log, record(t, p).Altered)
}
})
// Issue 774 (the lease write met at novox): the controller's service is stopped on the node placement places
// mesh-controller on, never on the lease's host (a hostname); none or two such nodes refuse by name, and a failed act
// is logged beside its condition.
t.Run("the controller's node", func(t *testing.T) {
for _, row := range []struct {
name, stop, said string
place func(f *leaseFake)
}{
{"placed on a", "a", "", func(*leaseFake) {}},
{"placed nowhere", "", "no node places mesh-controller", func(f *leaseFake) {
delete(f.states[grants.ReadPlacement], "placement.a")
f.assign("a", "nats")
f.assign("a", "mod-x")
}},
{"placed twice", "", "two or more nodes place mesh-controller (a, b)", func(f *leaseFake) { f.assign("b", grants.Controller) }},
} {
f, p := newLeaseFake(t)
if _, err := p.LeaseWrite("operator@shanks", "the controller goes"); err != nil {
t.Fatal(err)
}
p.Compose()
row.place(f)
if err := warrant(t, p, "yes"); err != nil {
t.Fatal(err)
}
stops := slices.DeleteFunc(slices.Clone(f.calls), func(c string) bool { return !strings.HasPrefix(c, grants.CallServiceStop+"@") })
r := record(t, p)
switch {
case row.stop != "" && (!slices.Equal(stops, []string{grants.CallServiceStop + "@" + row.stop}) || r.Node != row.stop):
t.Errorf("%s: stopped %v, node %q; want the stop on %s alone", row.name, stops, r.Node, row.stop)
case row.stop == "" && (len(stops) != 0 || r.Open != nil || len(r.Acts) != 1 || r.Acts[0].Step != grants.StepFailed ||
!strings.Contains(r.Acts[0].Said, row.said)):
t.Errorf("%s: stopped %v, %+v; want the act failed saying %q, nothing stopped", row.name, stops, r, row.said)
case row.stop == "" && !slices.ContainsFunc(f.logged, func(l string) bool { return strings.Contains(l, row.said) }):
t.Errorf("%s: logged %q, want a line saying %q", row.name, f.logged, row.said)
}
}
})
// 774/S19/AC1, AC2: hand-back restores the expiry, deletes holder and removal-due, and starts the controller only once
// the list with its rights is loaded; once the removal ran it is refused.
t.Run("hand-back", func(t *testing.T) {
f, p := handed(t)
if _, err := p.LeaseHandBack("operator@shanks", "it goes back"); err != nil {
t.Fatal(err)
}
p.Compose()
f.log = nil
if err := warrant(t, p, "yes"); err != nil {
t.Fatal(err)
}
r := record(t, p)
if f.expiry != 15*time.Second || f.states[grants.ReadLease][grants.LeaseKey] != nil || r.Cut || f.active != "inactive" {
t.Fatalf("after the warrant: expiry %s, holder %s, cut %v, controller %s", f.expiry, f.states[grants.ReadLease][grants.LeaseKey], r.Cut, f.active)
}
if _, _, found, _ := f.ownGet(grants.StateLease, grants.KeyRemovalDue); found {
t.Error("removal-due is not deleted")
}
f.loads(p)
p.Compose()
if r := record(t, p); f.active != "active" || r.Open != nil || r.Acts[len(r.Acts)-1].Step != grants.StepDone {
t.Errorf("once loaded: controller %s, %+v", f.active, r)
}
// AC2: once the removal ran, refused naming why.
f, p = handed(t)
p.RemovalDueHeard()
p.Compose()
if _, err := p.LeaseHandBack("operator@shanks", "too late"); err == nil || !strings.Contains(err.Error(), "without its consumer") {
t.Errorf("hand-back after the removal: %v", err)
}
})
// 774/S20/AC1, AC2, AC3: the marker of removal-due starts the removal; a failure says removal-failed and writes the key
// again with a day; a hand-back's delete removes nothing.
t.Run("the day-later removal", func(t *testing.T) {
f, p := handed(t)
f.log = nil
p.RemovalDueHeard()
p.Compose()
want := []string{"consumer removed", "hand-act removed the controller's consumer of DEAD_LETTER_NOTICES", "stream removed",
"hand-act removed the stream ASSIGNMENTS"}
if !slices.Equal(f.log, want) || record(t, p).Removed.IsZero() {
t.Errorf("the removal %v (removed %v), want %v", f.log, record(t, p).Removed, want)
}
f, p = handed(t)
f.log = nil
f.answer[grants.CallStreamRemove] = func(map[string]any) (any, error) {
return nil, errors.New("ASSIGNMENTS has 1 consumers: a reader is attached, so it was not removed")
}
p.RemovalDueHeard()
p.Compose()
if !slices.Contains(f.log, "event removal-failed") || f.ttls[grants.StateLease+"."+grants.KeyRemovalDue] != grants.RemovalAfter ||
!record(t, p).Removed.IsZero() {
t.Errorf("a failed removal: %v, removal-due %s", f.log, f.ttls[grants.StateLease+"."+grants.KeyRemovalDue])
}
f, p = handed(t)
if _, err := p.LeaseHandBack("operator@shanks", "it goes back"); err != nil {
t.Fatal(err)
}
p.Compose()
if err := warrant(t, p, "yes"); err != nil {
t.Fatal(err)
}
f.log = nil
p.RemovalDueHeard() // the hand-back's own delete of removal-due, heard
p.Compose()
if slices.ContainsFunc(f.log, func(s string) bool { return strings.Contains(s, "removed") }) {
t.Errorf("a hand-back's delete of removal-due removed %v", f.log)
}
})
// 774/S21/AC1: once the lease is handed away a membership is written to the state and not published on its subject.
t.Run("memberships after the write", func(t *testing.T) {
f, p := handed(t)
f.publishes = nil
f.assign("b", "mod-x")
p.Compose()
if len(f.publishes) != 0 {
t.Errorf("published on mesh.assignment after the lease write: %v", f.publishes)
}
if _, _, found, _ := f.ownGet(grants.StateMemberships, "b.mod-x"); !found {
t.Error("the membership is not written to the state")
}
})
}
// handed is a part whose lease write is done: the lease names mesh-grants and removal-due is written.
func handed(t *testing.T) (*leaseFake, *grants.Part) {
t.Helper()
f, p := newLeaseFake(t)
if _, err := p.LeaseWrite("operator@shanks", "the controller goes"); err != nil {
t.Fatal(err)
}
p.Compose()
if err := warrant(t, p, "yes"); err != nil {
t.Fatal(err)
}
f.loads(p)
p.Compose()
if r := record(t, p); r.Handed.IsZero() {
t.Fatalf("the write did not finish: %+v, %v", r, f.log)
}
return f, p
}
modules/mesh-grants/module.json
+3 -42
View File
@@ -2,16 +2,6 @@
"module": "mesh-grants",
"version": "1",
"slug": "grants",
"claims": [
{
"name": "mesh-grants",
"scope": "mesh",
"serves": [
"lease-write",
"lease-hand-back"
]
}
],
"reads": [
"mesh-config.resolved",
"mesh-delivery-assigner.wants",
@@ -21,16 +11,13 @@
"mesh-delivery-assigner.placement",
"node-engine.have",
"mesh-vault.bus-accounts",
"mesh-controller.lease",
"mesh-conditions.takeover",
"mesh-enrolment.enrolments",
"mesh-vault.node-logins"
],
"consumes": [
"mesh-conditions.condition-raised",
"mesh-conditions.condition-cleared",
"mesh-vault.account-issued",
"nats.bucket-expiry-set"
"mesh-vault.account-issued"
],
"uses": [
"operator-channel"
@@ -39,20 +26,7 @@
"nats.nats_streams",
"nats.nats_user_can",
"nats.nats_users",
"seat:node-engine.root-free",
"seat:mesh-hand-acts.warranted",
"seat:node-service-manager.stop",
"seat:node-service-manager.start",
"seat:node-service-manager.status",
"nats.nats_bucket_expiry",
"nats.nats_consumer_remove",
"nats.nats_stream_remove",
"seat:mesh-hand-acts.hand-act",
"mesh-delivery-view.node_trusts",
"seat:mesh-config.compare",
"seat:mesh-wants.upgrade",
"mesh-vault.vault_status",
"seat:issue-tracker.get"
"seat:node-engine.root-free"
],
"emits": [
"asserted",
@@ -67,14 +41,9 @@
"grants_user_list",
"grants_compose_now",
"grants_assert",
"grants_take_over",
"grants_take_over_state",
"grants_hand_back",
"grants_accounts",
"grants_consumer_match",
"grants_accept_hashes",
"grants_lease_check",
"grants_removal_due"
"grants_accept_hashes"
],
"state": [
{
@@ -89,18 +58,10 @@
"name": "grants",
"history": 5
},
{
"name": "lease",
"key-ttl": true
},
{
"name": "memberships",
"history": 5
},
{
"name": "takeover",
"history": 20
},
{
"name": "user-list",
"history": 20