package manifest import ( "encoding/json" "errors" "fmt" "slices" "strings" "testing" "time" "git.novox.be/novox/mesh-sdk/go/asks" "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/health" "git.novox.be/novox/mesh-sdk/go/seal" "mesh-grants/internal/grants" ) // leaseFake is the bus as the lease write meets it: fakeBus's states and tools, and the seats and tools the write asks. // Each answer is shaped as its writer's code answers it (issue 774): node-service-manager's status the systemd module's // Manager.Status (ActiveState), nats_bucket_expiry nats-tools' BucketExpirySet (now), nats_consumer_remove its // ConsumerRemoved (removed) and an absent consumer its error ("has no consumer"), nats_stream_remove its StreamRemoved // (removed, present), issue-tracker.get mesh-issues' Brief (status), node_trusts mesh-delivery-view's TrustsAnswer // (every), mesh-config's compare its Comparison (source, count), mesh-wants' upgrade compare the assigner's answer // (empty), vault_status mesh-vault's answer (ready, minting.on, minting.held_back). type leaseFake struct { *fakeBus answer map[string]func(map[string]any) (any, error) active string // the controller's ActiveState expiry time.Duration // the lease bucket's ttls map[string]time.Duration log []string // the write's acts on the bus, in order } func newLeaseFake(t *testing.T) (*leaseFake, *grants.Part) { t.Helper() b := fixture() b.manifestOf("nats", `{"module":"nats","version":"1","claims":[{"name":"mesh-broker"}],"own-secrets":{"broker":"/var/lib/nats/broker"},`+ `"tools":["nats_bucket_expiry","nats_consumer_remove","nats_stream_remove"]}`) b.put(grants.ReadLease, grants.LeaseKey, map[string]any{"instance": "controller@a pid 1", "host": "a", "epoch": 60, "taken": "2026-10-30T14:52:10Z"}) b.put(grants.ReadTakeover, health.TakeoverKey, health.Takeover{V: 1, Follow: health.FollowKeeper, By: "operator", Why: "the keeper took over"}) busKey, err := seal.GenerateKeyPair() if err != nil { t.Fatal(err) } b.put(grants.ReadHave, "a.nats", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK, UserList: &delivery.UserListWritten{Source: "a.nats.src2", Revision: 7, Digest: "sha256:aa", Users: 6, Key: busKey.PublicKey}}) // The controller's hash in the vault, so the list carries the controller and the cut changes it. b.put(grants.ReadAccounts, buscompose.ControllerName, map[string]any{"hash": "$2a$11$controller", "generation": 1, "at": "2026-10-30T14:52:10Z"}) f := &leaseFake{fakeBus: b, active: "active", expiry: 15 * time.Second, ttls: map[string]time.Duration{}} f.answer = map[string]func(map[string]any) (any, error){ grants.CallIssue: func(map[string]any) (any, error) { return map[string]any{"status": "resolved"}, nil }, grants.CallNodeTrusts: func(map[string]any) (any, error) { return map[string]any{"every": true, "not_yet": []string{}}, nil }, grants.CallConfigCompare: func(map[string]any) (any, error) { return map[string]any{"source": "indexer", "count": 0}, nil }, grants.CallUpgrade: func(map[string]any) (any, error) { return map[string]any{"empty": true, "splits": []any{}}, nil }, grants.CallVaultStatus: func(map[string]any) (any, error) { return map[string]any{"ready": true, "minting": map[string]any{"on": true, "held_back": map[string]string{}}}, nil }, grants.CallServiceStop: func(map[string]any) (any, error) { f.active = "inactive"; return map[string]any{"ok": true}, nil }, grants.CallServiceStart: func(map[string]any) (any, error) { f.active = "active"; return map[string]any{"ok": true}, nil }, grants.CallServiceStatus: func(map[string]any) (any, error) { return map[string]any{"ActiveState": f.active}, nil }, grants.CallBucketExpiry: func(a map[string]any) (any, error) { now := fmt.Sprint(a["max_age"]) d, _ := time.ParseDuration(strings.Replace(now, "none", "0s", 1)) f.expiry = d f.log = append(f.log, "expiry "+now) return map[string]any{"bucket": a["bucket"], "now": now, "changed": true}, nil }, grants.CallRecordWarrant: func(a map[string]any) (any, error) { f.log = append(f.log, "warranted"); return map[string]any{}, nil }, grants.CallRecordHandAct: func(a map[string]any) (any, error) { f.log = append(f.log, "hand-act "+fmt.Sprint(a["what"])) return map[string]any{}, nil }, grants.CallConsumerRemove: func(map[string]any) (any, error) { f.log = append(f.log, "consumer removed") return map[string]any{"removed": true}, nil }, grants.CallStreamRemove: func(map[string]any) (any, error) { f.log = append(f.log, "stream removed") return map[string]any{"removed": true, "present": true}, nil }, } p := partOn(b) p.Call, p.Store, p.Lease, p.Asker, p.Objects = f, ttlStore{ownStore{b}, f}, f, &fakeAsker{wheres: map[string]*asks.Warrant{}}, &fakeObjects{} p.SaySeat = func(event string, _ any) error { f.log = append(f.log, "event "+event); return nil } p.Nonce = func() string { return "0001" } return f, p } func (f *leaseFake) Call(key string, body any) (json.RawMessage, error) { base, _, _ := strings.Cut(key, "@") if answer, ok := f.answer[base]; ok { f.mu.Lock() f.calls = append(f.calls, key) f.mu.Unlock() args, _ := body.(map[string]any) out, err := answer(args) if err != nil { return nil, err } return json.Marshal(out) } return f.fakeBus.Call(key, body) } func (f *leaseFake) PutHolder(value []byte) error { f.log = append(f.log, "holder") f.states[grants.ReadLease][grants.LeaseKey] = value return nil } func (f *leaseFake) DeleteHolder() error { f.log = append(f.log, "holder deleted") delete(f.states[grants.ReadLease], grants.LeaseKey) return nil } func (f *leaseFake) Expiry() (time.Duration, error) { return f.expiry, nil } // loads says the bus node loaded the list the part last sealed. func (f *leaseFake) loads(p *grants.Part) { g, _, _ := p.Sealed() f.put(grants.ReadHave, "a.nats", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK, UserList: &delivery.UserListWritten{Source: delivery.GrantsUserListSource, Revision: g.Revision, Digest: g.Digest, Users: g.Users, Key: keyOf(f)}}) } func keyOf(f *leaseFake) string { var h delivery.Have _ = json.Unmarshal(f.states[grants.ReadHave]["a.nats"], &h) return h.UserList.Key } // ttlStore is the part's own store with a time to live per key, recorded. type ttlStore struct { ownStore f *leaseFake } func (s ttlStore) PutTTL(state, key string, value []byte, ttl time.Duration) (uint64, error) { s.f.ttls[state+"."+key] = ttl s.f.log = append(s.f.log, key+" "+ttl.String()) return s.Put(state, key, value) } // record is the part's lease record now. func record(t *testing.T, p *grants.Part) grants.LeaseRecord { t.Helper() r, err := p.LeaseState() if err != nil { t.Fatal(err) } return r } // warrant answers the open act's ask with option, as the router's warrant hands it. func warrant(t *testing.T, p *grants.Part, option string) error { t.Helper() r := record(t, p) if r.Open == nil { t.Fatal("no act is open") } return p.Warranted(asks.Option{ID: option}, asks.Warrant{Ask: r.Open.Ask, Option: option, Outcome: asks.OutcomeChosen}) } // Issue 774 (novox/hq ADR 0582): the lease write, from its ask through the day-later removal, against stand-in seats. func TestTheLeaseWrite(t *testing.T) { // 774/S14/AC1: any one precondition not green refuses the verb naming it; all green asks. t.Run("preconditions", func(t *testing.T) { for _, row := range []struct { name string break_ func(f *leaseFake) }{ {"lease-check", func(f *leaseFake) { f.manifestOf("mod-u", `{"module":"mod-u","version":"1","reads":["mesh-controller.conditions"]}`) }}, {"earlier-steps", func(f *leaseFake) { f.answer[grants.CallIssue] = func(a map[string]any) (any, error) { if fmt.Sprint(a["number"]) == "1954" { return map[string]any{"status": "fixing"}, nil } return map[string]any{"status": "resolved"}, nil } }}, {"node-trusts", func(f *leaseFake) { f.answer[grants.CallNodeTrusts] = func(map[string]any) (any, error) { return map[string]any{"every": false, "not_yet": []string{"ace"}}, nil } }}, {"config-compare", func(f *leaseFake) { f.answer[grants.CallConfigCompare] = func(map[string]any) (any, error) { return map[string]any{"source": "indexer", "count": 2}, nil } }}, {"upgrade-compare", func(f *leaseFake) { f.answer[grants.CallUpgrade] = func(map[string]any) (any, error) { return map[string]any{"empty": false, "splits": []any{1}}, nil } }}, {"keeper-take-over", func(f *leaseFake) { f.put(grants.ReadTakeover, health.TakeoverKey, health.Takeover{V: 1, Follow: health.FollowController}) }}, {"vault-minting", func(f *leaseFake) { f.answer[grants.CallVaultStatus] = func(map[string]any) (any, error) { return map[string]any{"ready": true, "minting": map[string]any{"on": true, "held_back": map[string]string{"a.x": "why"}}}, nil } }}, {"bus-tools", func(f *leaseFake) { f.manifestOf("nats", natsManifest) }}, {"", func(*leaseFake) {}}, } { f, p := newLeaseFake(t) row.break_(f) _, err := p.LeaseWrite("operator@shanks", "the controller goes") switch { case row.name == "" && err != nil: t.Errorf("all green: refused: %v", err) case row.name != "" && (err == nil || !strings.Contains(err.Error(), row.name+": ")): t.Errorf("%s not green: %v, want a refusal naming it", row.name, err) case row.name != "": for _, other := range grants.LeaseCheckNames { if other != row.name && strings.Contains(err.Error(), other+": ") { t.Errorf("%s not green: the refusal names %s too: %v", row.name, other, err) } } } if r := record(t, p); (r.Open != nil) != (row.name == "") { t.Errorf("%q: an act open %v", row.name, r.Open) } } }) // 774/S15/AC1, AC2: a no, an expiry, or a warrant whose checks changed writes nothing, naming the changed check. t.Run("the ask and the warrant", func(t *testing.T) { for _, row := range []struct { name, step, said string answer func(t *testing.T, f *leaseFake, p *grants.Part) }{ {"declined", grants.StepDeclined, "said no", func(t *testing.T, _ *leaseFake, p *grants.Part) { if err := warrant(t, p, "no"); err != nil { t.Fatal(err) } }}, {"expired", "expired", "expired", func(t *testing.T, _ *leaseFake, p *grants.Part) { r := record(t, p) p.Said("", asks.Warrant{Ask: r.Open.Ask, Outcome: asks.OutcomeExpired}, nil) }}, {"checks changed", grants.StepRefused, "the checks changed since the ask: vault-minting (now: ready true, minting true, 1 held back)", func(t *testing.T, f *leaseFake, p *grants.Part) { // The vault holds one account back between the ask and the warrant. f.answer[grants.CallVaultStatus] = func(map[string]any) (any, error) { return map[string]any{"ready": true, "minting": map[string]any{"on": true, "held_back": map[string]string{"a.x": "why"}}}, nil } if err := warrant(t, p, "yes"); err != nil { t.Fatal(err) } }}, } { f, p := newLeaseFake(t) if _, err := p.LeaseWrite("operator@shanks", "the controller goes"); err != nil { t.Fatal(err) } p.Compose() // the process asks it if a := p.Asker.(*fakeAsker); len(a.asks) != 1 || len(a.asks[0].Options) != 2 || a.asks[0].Options[0].Binds == "" { t.Fatalf("%s: asked %+v, want one ask whose yes binds the act", row.name, a.asks) } row.answer(t, f, p) r := record(t, p) if r.Open != nil || len(r.Acts) != 1 || r.Acts[0].Step != row.step || !strings.Contains(r.Acts[0].Said, row.said) { t.Errorf("%s: open %v, acts %+v; want one act %s saying %q", row.name, r.Open, r.Acts, row.step, row.said) } if f.count(grants.CallServiceStop+"@a") != 0 || len(f.log) != 0 || r.Cut { t.Errorf("%s: something was written: stops %d, %v, cut %v", row.name, f.count(grants.CallServiceStop+"@a"), f.log, r.Cut) } } }) // 774/S16/AC1, AC2; 774/S17/AC1, AC2: the write's six steps, in order, each read back before the next. t.Run("the write", func(t *testing.T) { f, p := newLeaseFake(t) f.answer[grants.CallServiceStop] = func(map[string]any) (any, error) { return map[string]any{"ok": true}, nil } // reads back active if _, err := p.LeaseWrite("operator@shanks", "the controller goes"); err != nil { t.Fatal(err) } p.Compose() if err := warrant(t, p, "yes"); err != nil { t.Fatal(err) } if r := record(t, p); r.Open != nil || r.Cut || r.Acts[0].Step != grants.StepFailed || !strings.Contains(r.Acts[0].Said, "did not read back stopped") || len(f.log) != 0 { t.Fatalf("a stop not read back: %+v, %v; want the act failed there, nothing written", r, f.log) } f, p = newLeaseFake(t) if _, err := p.LeaseWrite("operator@shanks", "the controller goes"); err != nil { t.Fatal(err) } p.Compose() before, _, _ := p.Sealed() if err := warrant(t, p, "yes"); err != nil { t.Fatal(err) } r := record(t, p) g, _, _ := p.Sealed() if f.active != "inactive" || !r.Cut || r.Open == nil || r.Open.Step != grants.StepCut || r.Open.List != g.Digest || g.Digest == before.Digest { t.Fatalf("after the warrant: active %s, %+v, sealed %s (was %s); want stopped, cut and waiting on the cut list", f.active, r, g.Digest, before.Digest) } if len(f.log) != 0 { t.Fatalf("the expiry or the key was touched before the cut list loaded: %v", f.log) } // The cut list takes the controller's write on the lease and gives mesh-grants the key holder. var st grants.UserListState raw, _, _, _ := ownStore{f.fakeBus}.Get(grants.StateUserList, grants.KeyList) _ = json.Unmarshal(raw, &st) p.Compose() // a pass with the bus node not loaded yet waits if len(f.log) != 0 { t.Fatalf("a pass before the load acted: %v", f.log) } f.loads(p) p.Compose() want := []string{"expiry none", "holder", "event lease-handed", "warranted", "removal-due 24h0m0s"} if !slices.Equal(f.log, want) { t.Errorf("the write's acts %v, want %v in that order", f.log, want) } var h buscompose.LeaseHolder _ = json.Unmarshal(f.states[grants.ReadLease][grants.LeaseKey], &h) r = record(t, p) if h.HandedTo != "mesh-grants" || h.HandedBy != "operator" || h.Why != "the controller goes" || h.Ask != r.Ask || h.At.IsZero() || r.Open != nil || r.Handed.IsZero() || r.Acts[len(r.Acts)-1].Step != grants.StepDone { t.Errorf("holder %+v, record %+v", h, r) } p.Compose() // the pass the lease key's change wakes if g, _, _ := p.Sealed(); !g.LeaseHanded { t.Error("the user-list key does not say the lease is handed") } if _, err := p.HandBack("operator@shanks", "back"); err == nil || !strings.HasPrefix(err.Error(), "gone") { t.Errorf("the user list's hand-back once the lease is handed: %v, want gone", err) } if _, err := p.LeaseWrite("operator@shanks", "again"); err == nil { t.Error("a second lease-write with the lease handed was asked") } // 774/S18/AC1: the key or the expiry changed by another hand is said once, and nothing is written. f.log = nil f.put(grants.ReadLease, grants.LeaseKey, map[string]any{"instance": "controller@a pid 9", "host": "a", "epoch": 61}) p.Compose() p.Compose() f.expiry = 15 * time.Second p.Compose() if !slices.Equal(f.log, []string{"event lease-altered"}) || !strings.Contains(record(t, p).Altered, "controller@a pid 9") { t.Errorf("an altered lease said %v (%q), want lease-altered once naming it", f.log, record(t, p).Altered) } }) // 774/S19/AC1, AC2: hand-back restores the expiry, deletes holder and removal-due, and starts the controller only once // the list with its rights is loaded; once the removal ran it is refused. t.Run("hand-back", func(t *testing.T) { f, p := handed(t) if _, err := p.LeaseHandBack("operator@shanks", "it goes back"); err != nil { t.Fatal(err) } p.Compose() f.log = nil if err := warrant(t, p, "yes"); err != nil { t.Fatal(err) } r := record(t, p) if f.expiry != 15*time.Second || f.states[grants.ReadLease][grants.LeaseKey] != nil || r.Cut || f.active != "inactive" { t.Fatalf("after the warrant: expiry %s, holder %s, cut %v, controller %s", f.expiry, f.states[grants.ReadLease][grants.LeaseKey], r.Cut, f.active) } if _, _, found, _ := f.ownGet(grants.StateLease, grants.KeyRemovalDue); found { t.Error("removal-due is not deleted") } f.loads(p) p.Compose() if r := record(t, p); f.active != "active" || r.Open != nil || r.Acts[len(r.Acts)-1].Step != grants.StepDone { t.Errorf("once loaded: controller %s, %+v", f.active, r) } // AC2: once the removal ran, refused naming why. f, p = handed(t) p.RemovalDueHeard() p.Compose() if _, err := p.LeaseHandBack("operator@shanks", "too late"); err == nil || !strings.Contains(err.Error(), "without its consumer") { t.Errorf("hand-back after the removal: %v", err) } }) // 774/S20/AC1, AC2, AC3: the marker of removal-due starts the removal; a failure says removal-failed and writes the key // again with a day; a hand-back's delete removes nothing. t.Run("the day-later removal", func(t *testing.T) { f, p := handed(t) f.log = nil p.RemovalDueHeard() p.Compose() want := []string{"consumer removed", "hand-act removed the controller's consumer of DEAD_LETTER_NOTICES", "stream removed", "hand-act removed the stream ASSIGNMENTS"} if !slices.Equal(f.log, want) || record(t, p).Removed.IsZero() { t.Errorf("the removal %v (removed %v), want %v", f.log, record(t, p).Removed, want) } f, p = handed(t) f.log = nil f.answer[grants.CallStreamRemove] = func(map[string]any) (any, error) { return nil, errors.New("ASSIGNMENTS has 1 consumers: a reader is attached, so it was not removed") } p.RemovalDueHeard() p.Compose() if !slices.Contains(f.log, "event removal-failed") || f.ttls[grants.StateLease+"."+grants.KeyRemovalDue] != grants.RemovalAfter || !record(t, p).Removed.IsZero() { t.Errorf("a failed removal: %v, removal-due %s", f.log, f.ttls[grants.StateLease+"."+grants.KeyRemovalDue]) } f, p = handed(t) if _, err := p.LeaseHandBack("operator@shanks", "it goes back"); err != nil { t.Fatal(err) } p.Compose() if err := warrant(t, p, "yes"); err != nil { t.Fatal(err) } f.log = nil p.RemovalDueHeard() // the hand-back's own delete of removal-due, heard p.Compose() if slices.ContainsFunc(f.log, func(s string) bool { return strings.Contains(s, "removed") }) { t.Errorf("a hand-back's delete of removal-due removed %v", f.log) } }) // 774/S21/AC1: once the lease is handed away a membership is written to the state and not published on its subject. t.Run("memberships after the write", func(t *testing.T) { f, p := handed(t) f.publishes = nil f.assign("b", "mod-x") p.Compose() if len(f.publishes) != 0 { t.Errorf("published on mesh.assignment after the lease write: %v", f.publishes) } if _, _, found, _ := f.ownGet(grants.StateMemberships, "b.mod-x"); !found { t.Error("the membership is not written to the state") } }) } // handed is a part whose lease write is done: the lease names mesh-grants and removal-due is written. func handed(t *testing.T) (*leaseFake, *grants.Part) { t.Helper() f, p := newLeaseFake(t) if _, err := p.LeaseWrite("operator@shanks", "the controller goes"); err != nil { t.Fatal(err) } p.Compose() if err := warrant(t, p, "yes"); err != nil { t.Fatal(err) } f.loads(p) p.Compose() if r := record(t, p); r.Handed.IsZero() { t.Fatalf("the write did not finish: %+v, %v", r, f.log) } return f, p }