jschoubben
7cee3842b8
The lease write addressed node-service-manager at the lease holder's host, a
hostname no node carries ("novox-be" on novox), so the operator's yes timed out
at its first step and failed without a log line. The node now comes from
placement (exactly one node placing mesh-controller, else refused by name), a
failed act is logged beside its condition, and the asks record is read from
messenger_asks, the bucket the composed grant names, instead of asks, which no
grant allows. The fixtures give the lease a host unlike the node, so the tests
no longer agree with the mistake. (issue 774)
Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
504 lines
22 KiB
Go
504 lines
22 KiB
Go
package manifest
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/asks"
|
|
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
"git.novox.be/novox/mesh-sdk/go/health"
|
|
"git.novox.be/novox/mesh-sdk/go/seal"
|
|
|
|
"mesh-grants/internal/grants"
|
|
)
|
|
|
|
// leaseFake is the bus as the lease write meets it: fakeBus's states and tools, and the seats and tools the write asks.
|
|
// Each answer is shaped as its writer's code answers it (issue 774): node-service-manager's status the systemd module's
|
|
// Manager.Status (ActiveState), nats_bucket_expiry nats-tools' BucketExpirySet (now), nats_consumer_remove its
|
|
// ConsumerRemoved (removed) and an absent consumer its error ("has no consumer"), nats_stream_remove its StreamRemoved
|
|
// (removed, present), issue-tracker.get mesh-issues' Brief (status), node_trusts mesh-delivery-view's TrustsAnswer
|
|
// (every), mesh-config's compare its Comparison (source, count), mesh-wants' upgrade compare the assigner's answer
|
|
// (empty), vault_status mesh-vault's answer (ready, minting.on, minting.held_back).
|
|
type leaseFake struct {
|
|
*fakeBus
|
|
answer map[string]func(map[string]any) (any, error)
|
|
active string // the controller's ActiveState
|
|
expiry time.Duration // the lease bucket's
|
|
ttls map[string]time.Duration
|
|
log []string // the write's acts on the bus, in order
|
|
logged []string // the process's log lines
|
|
}
|
|
|
|
func newLeaseFake(t *testing.T) (*leaseFake, *grants.Part) {
|
|
t.Helper()
|
|
b := fixture()
|
|
b.manifestOf("nats", `{"module":"nats","version":"1","claims":[{"name":"mesh-broker"}],"own-secrets":{"broker":"/var/lib/nats/broker"},`+
|
|
`"tools":["nats_bucket_expiry","nats_consumer_remove","nats_stream_remove"]}`)
|
|
// The lease's host is the machine's hostname, never a node's name (issue 774: "novox-be" on the node novox); the
|
|
// controller's node is the one placement places it on.
|
|
b.put(grants.ReadLease, grants.LeaseKey, map[string]any{"instance": "controller@box-a pid 1", "host": "box-a", "epoch": 60,
|
|
"taken": "2026-10-30T14:52:10Z"})
|
|
b.assign("a", grants.Controller)
|
|
b.put(grants.ReadTakeover, health.TakeoverKey, health.Takeover{V: 1, Follow: health.FollowKeeper, By: "operator", Why: "the keeper took over"})
|
|
busKey, err := seal.GenerateKeyPair()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
b.put(grants.ReadHave, "a.nats", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK,
|
|
UserList: &delivery.UserListWritten{Source: "a.nats.src2", Revision: 7, Digest: "sha256:aa", Users: 6, Key: busKey.PublicKey}})
|
|
// The controller's hash in the vault, so the list carries the controller and the cut changes it.
|
|
b.put(grants.ReadAccounts, buscompose.ControllerName, map[string]any{"hash": "$2a$11$controller", "generation": 1,
|
|
"at": "2026-10-30T14:52:10Z"})
|
|
f := &leaseFake{fakeBus: b, active: "active", expiry: 15 * time.Second, ttls: map[string]time.Duration{}}
|
|
f.answer = map[string]func(map[string]any) (any, error){
|
|
grants.CallIssue: func(map[string]any) (any, error) { return map[string]any{"status": "resolved"}, nil },
|
|
grants.CallNodeTrusts: func(map[string]any) (any, error) { return map[string]any{"every": true, "not_yet": []string{}}, nil },
|
|
grants.CallConfigCompare: func(map[string]any) (any, error) { return map[string]any{"source": "indexer", "count": 0}, nil },
|
|
grants.CallUpgrade: func(map[string]any) (any, error) { return map[string]any{"empty": true, "splits": []any{}}, nil },
|
|
grants.CallVaultStatus: func(map[string]any) (any, error) {
|
|
return map[string]any{"ready": true, "minting": map[string]any{"on": true, "held_back": map[string]string{}}}, nil
|
|
},
|
|
grants.CallServiceStop: func(map[string]any) (any, error) { f.active = "inactive"; return map[string]any{"ok": true}, nil },
|
|
grants.CallServiceStart: func(map[string]any) (any, error) { f.active = "active"; return map[string]any{"ok": true}, nil },
|
|
grants.CallServiceStatus: func(map[string]any) (any, error) { return map[string]any{"ActiveState": f.active}, nil },
|
|
grants.CallBucketExpiry: func(a map[string]any) (any, error) {
|
|
now := fmt.Sprint(a["max_age"])
|
|
d, _ := time.ParseDuration(strings.Replace(now, "none", "0s", 1))
|
|
f.expiry = d
|
|
f.log = append(f.log, "expiry "+now)
|
|
return map[string]any{"bucket": a["bucket"], "now": now, "changed": true}, nil
|
|
},
|
|
grants.CallRecordWarrant: func(a map[string]any) (any, error) { f.log = append(f.log, "warranted"); return map[string]any{}, nil },
|
|
grants.CallRecordHandAct: func(a map[string]any) (any, error) {
|
|
f.log = append(f.log, "hand-act "+fmt.Sprint(a["what"]))
|
|
return map[string]any{}, nil
|
|
},
|
|
grants.CallConsumerRemove: func(map[string]any) (any, error) {
|
|
f.log = append(f.log, "consumer removed")
|
|
return map[string]any{"removed": true}, nil
|
|
},
|
|
grants.CallStreamRemove: func(map[string]any) (any, error) {
|
|
f.log = append(f.log, "stream removed")
|
|
return map[string]any{"removed": true, "present": true}, nil
|
|
},
|
|
}
|
|
p := partOn(b)
|
|
p.Call, p.Store, p.Lease, p.Asker, p.Objects = f, ttlStore{ownStore{b}, f}, f, &fakeAsker{wheres: map[string]*asks.Warrant{}}, &fakeObjects{}
|
|
p.SaySeat = func(event string, _ any) error { f.log = append(f.log, "event "+event); return nil }
|
|
p.Nonce = func() string { return "0001" }
|
|
p.Logf = func(format string, a ...any) { f.logged = append(f.logged, fmt.Sprintf(format, a...)) }
|
|
return f, p
|
|
}
|
|
|
|
func (f *leaseFake) Call(key string, body any) (json.RawMessage, error) {
|
|
base, _, _ := strings.Cut(key, "@")
|
|
if answer, ok := f.answer[base]; ok {
|
|
f.mu.Lock()
|
|
f.calls = append(f.calls, key)
|
|
f.mu.Unlock()
|
|
args, _ := body.(map[string]any)
|
|
out, err := answer(args)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return json.Marshal(out)
|
|
}
|
|
return f.fakeBus.Call(key, body)
|
|
}
|
|
|
|
func (f *leaseFake) PutHolder(value []byte) error {
|
|
f.log = append(f.log, "holder")
|
|
f.states[grants.ReadLease][grants.LeaseKey] = value
|
|
return nil
|
|
}
|
|
|
|
func (f *leaseFake) DeleteHolder() error {
|
|
f.log = append(f.log, "holder deleted")
|
|
delete(f.states[grants.ReadLease], grants.LeaseKey)
|
|
return nil
|
|
}
|
|
|
|
func (f *leaseFake) Expiry() (time.Duration, error) { return f.expiry, nil }
|
|
|
|
// loads says the bus node loaded the list the part last sealed.
|
|
func (f *leaseFake) loads(p *grants.Part) {
|
|
g, _, _ := p.Sealed()
|
|
f.put(grants.ReadHave, "a.nats", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK,
|
|
UserList: &delivery.UserListWritten{Source: delivery.GrantsUserListSource, Revision: g.Revision, Digest: g.Digest,
|
|
Users: g.Users, Key: keyOf(f)}})
|
|
}
|
|
|
|
func keyOf(f *leaseFake) string {
|
|
var h delivery.Have
|
|
_ = json.Unmarshal(f.states[grants.ReadHave]["a.nats"], &h)
|
|
return h.UserList.Key
|
|
}
|
|
|
|
// ttlStore is the part's own store with a time to live per key, recorded.
|
|
type ttlStore struct {
|
|
ownStore
|
|
f *leaseFake
|
|
}
|
|
|
|
func (s ttlStore) PutTTL(state, key string, value []byte, ttl time.Duration) (uint64, error) {
|
|
s.f.ttls[state+"."+key] = ttl
|
|
s.f.log = append(s.f.log, key+" "+ttl.String())
|
|
return s.Put(state, key, value)
|
|
}
|
|
|
|
// record is the part's lease record now.
|
|
func record(t *testing.T, p *grants.Part) grants.LeaseRecord {
|
|
t.Helper()
|
|
r, err := p.LeaseState()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return r
|
|
}
|
|
|
|
// warrant answers the open act's ask with option, as the router's warrant hands it.
|
|
func warrant(t *testing.T, p *grants.Part, option string) error {
|
|
t.Helper()
|
|
r := record(t, p)
|
|
if r.Open == nil {
|
|
t.Fatal("no act is open")
|
|
}
|
|
return p.Warranted(asks.Option{ID: option}, asks.Warrant{Ask: r.Open.Ask, Option: option, Outcome: asks.OutcomeChosen})
|
|
}
|
|
|
|
// Issue 774 (novox/hq ADR 0582): the lease write, from its ask through the day-later removal, against stand-in seats.
|
|
func TestTheLeaseWrite(t *testing.T) {
|
|
// 774/S14/AC1: any one precondition not green refuses the verb naming it; all green asks.
|
|
t.Run("preconditions", func(t *testing.T) {
|
|
for _, row := range []struct {
|
|
name string
|
|
break_ func(f *leaseFake)
|
|
}{
|
|
{"lease-check", func(f *leaseFake) {
|
|
f.manifestOf("mod-u", `{"module":"mod-u","version":"1","reads":["mesh-controller.conditions"]}`)
|
|
}},
|
|
{"earlier-steps", func(f *leaseFake) {
|
|
f.answer[grants.CallIssue] = func(a map[string]any) (any, error) {
|
|
if fmt.Sprint(a["number"]) == "1954" {
|
|
return map[string]any{"status": "fixing"}, nil
|
|
}
|
|
return map[string]any{"status": "resolved"}, nil
|
|
}
|
|
}},
|
|
{"node-trusts", func(f *leaseFake) {
|
|
f.answer[grants.CallNodeTrusts] = func(map[string]any) (any, error) {
|
|
return map[string]any{"every": false, "not_yet": []string{"ace"}}, nil
|
|
}
|
|
}},
|
|
{"config-compare", func(f *leaseFake) {
|
|
f.answer[grants.CallConfigCompare] = func(map[string]any) (any, error) { return map[string]any{"source": "indexer", "count": 2}, nil }
|
|
}},
|
|
{"upgrade-compare", func(f *leaseFake) {
|
|
f.answer[grants.CallUpgrade] = func(map[string]any) (any, error) { return map[string]any{"empty": false, "splits": []any{1}}, nil }
|
|
}},
|
|
{"keeper-take-over", func(f *leaseFake) {
|
|
f.put(grants.ReadTakeover, health.TakeoverKey, health.Takeover{V: 1, Follow: health.FollowController})
|
|
}},
|
|
{"vault-minting", func(f *leaseFake) {
|
|
f.answer[grants.CallVaultStatus] = func(map[string]any) (any, error) {
|
|
return map[string]any{"ready": true, "minting": map[string]any{"on": true, "held_back": map[string]string{"a.x": "why"}}}, nil
|
|
}
|
|
}},
|
|
{"bus-tools", func(f *leaseFake) { f.manifestOf("nats", natsManifest) }},
|
|
{"", func(*leaseFake) {}},
|
|
} {
|
|
f, p := newLeaseFake(t)
|
|
row.break_(f)
|
|
_, err := p.LeaseWrite("operator@shanks", "the controller goes")
|
|
switch {
|
|
case row.name == "" && err != nil:
|
|
t.Errorf("all green: refused: %v", err)
|
|
case row.name != "" && (err == nil || !strings.Contains(err.Error(), row.name+": ")):
|
|
t.Errorf("%s not green: %v, want a refusal naming it", row.name, err)
|
|
case row.name != "":
|
|
for _, other := range grants.LeaseCheckNames {
|
|
if other != row.name && strings.Contains(err.Error(), other+": ") {
|
|
t.Errorf("%s not green: the refusal names %s too: %v", row.name, other, err)
|
|
}
|
|
}
|
|
}
|
|
if r := record(t, p); (r.Open != nil) != (row.name == "") {
|
|
t.Errorf("%q: an act open %v", row.name, r.Open)
|
|
}
|
|
}
|
|
})
|
|
|
|
// 774/S15/AC1, AC2: a no, an expiry, or a warrant whose checks changed writes nothing, naming the changed check.
|
|
t.Run("the ask and the warrant", func(t *testing.T) {
|
|
for _, row := range []struct {
|
|
name, step, said string
|
|
answer func(t *testing.T, f *leaseFake, p *grants.Part)
|
|
}{
|
|
{"declined", grants.StepDeclined, "said no", func(t *testing.T, _ *leaseFake, p *grants.Part) {
|
|
if err := warrant(t, p, "no"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}},
|
|
{"expired", "expired", "expired", func(t *testing.T, _ *leaseFake, p *grants.Part) {
|
|
r := record(t, p)
|
|
p.Said("", asks.Warrant{Ask: r.Open.Ask, Outcome: asks.OutcomeExpired}, nil)
|
|
}},
|
|
{"checks changed", grants.StepRefused, "the checks changed since the ask: vault-minting (now: ready true, minting true, 1 held back)",
|
|
func(t *testing.T, f *leaseFake, p *grants.Part) {
|
|
// The vault holds one account back between the ask and the warrant.
|
|
f.answer[grants.CallVaultStatus] = func(map[string]any) (any, error) {
|
|
return map[string]any{"ready": true, "minting": map[string]any{"on": true, "held_back": map[string]string{"a.x": "why"}}}, nil
|
|
}
|
|
if err := warrant(t, p, "yes"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}},
|
|
} {
|
|
f, p := newLeaseFake(t)
|
|
if _, err := p.LeaseWrite("operator@shanks", "the controller goes"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
p.Compose() // the process asks it
|
|
if a := p.Asker.(*fakeAsker); len(a.asks) != 1 || len(a.asks[0].Options) != 2 || a.asks[0].Options[0].Binds == "" {
|
|
t.Fatalf("%s: asked %+v, want one ask whose yes binds the act", row.name, a.asks)
|
|
}
|
|
row.answer(t, f, p)
|
|
r := record(t, p)
|
|
if r.Open != nil || len(r.Acts) != 1 || r.Acts[0].Step != row.step || !strings.Contains(r.Acts[0].Said, row.said) {
|
|
t.Errorf("%s: open %v, acts %+v; want one act %s saying %q", row.name, r.Open, r.Acts, row.step, row.said)
|
|
}
|
|
if f.count(grants.CallServiceStop+"@a") != 0 || len(f.log) != 0 || r.Cut {
|
|
t.Errorf("%s: something was written: stops %d, %v, cut %v", row.name, f.count(grants.CallServiceStop+"@a"), f.log, r.Cut)
|
|
}
|
|
}
|
|
})
|
|
|
|
// 774/S16/AC1, AC2; 774/S17/AC1, AC2: the write's six steps, in order, each read back before the next.
|
|
t.Run("the write", func(t *testing.T) {
|
|
f, p := newLeaseFake(t)
|
|
f.answer[grants.CallServiceStop] = func(map[string]any) (any, error) { return map[string]any{"ok": true}, nil } // reads back active
|
|
if _, err := p.LeaseWrite("operator@shanks", "the controller goes"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
p.Compose()
|
|
if err := warrant(t, p, "yes"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if r := record(t, p); r.Open != nil || r.Cut || r.Acts[0].Step != grants.StepFailed || !strings.Contains(r.Acts[0].Said, "did not read back stopped") || len(f.log) != 0 {
|
|
t.Fatalf("a stop not read back: %+v, %v; want the act failed there, nothing written", r, f.log)
|
|
}
|
|
|
|
f, p = newLeaseFake(t)
|
|
if _, err := p.LeaseWrite("operator@shanks", "the controller goes"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
p.Compose()
|
|
before, _, _ := p.Sealed()
|
|
if err := warrant(t, p, "yes"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r := record(t, p)
|
|
g, _, _ := p.Sealed()
|
|
if f.active != "inactive" || !r.Cut || r.Open == nil || r.Open.Step != grants.StepCut || r.Open.List != g.Digest || g.Digest == before.Digest {
|
|
t.Fatalf("after the warrant: active %s, %+v, sealed %s (was %s); want stopped, cut and waiting on the cut list", f.active, r, g.Digest, before.Digest)
|
|
}
|
|
if len(f.log) != 0 {
|
|
t.Fatalf("the expiry or the key was touched before the cut list loaded: %v", f.log)
|
|
}
|
|
// The cut list takes the controller's write on the lease and gives mesh-grants the key holder.
|
|
var st grants.UserListState
|
|
raw, _, _, _ := ownStore{f.fakeBus}.Get(grants.StateUserList, grants.KeyList)
|
|
_ = json.Unmarshal(raw, &st)
|
|
p.Compose() // a pass with the bus node not loaded yet waits
|
|
if len(f.log) != 0 {
|
|
t.Fatalf("a pass before the load acted: %v", f.log)
|
|
}
|
|
f.loads(p)
|
|
p.Compose()
|
|
want := []string{"expiry none", "holder", "event lease-handed", "warranted", "removal-due 24h0m0s"}
|
|
if !slices.Equal(f.log, want) {
|
|
t.Errorf("the write's acts %v, want %v in that order", f.log, want)
|
|
}
|
|
var h buscompose.LeaseHolder
|
|
_ = json.Unmarshal(f.states[grants.ReadLease][grants.LeaseKey], &h)
|
|
r = record(t, p)
|
|
if h.HandedTo != "mesh-grants" || h.HandedBy != "operator" || h.Why != "the controller goes" || h.Ask != r.Ask || h.At.IsZero() ||
|
|
r.Open != nil || r.Handed.IsZero() || r.Acts[len(r.Acts)-1].Step != grants.StepDone {
|
|
t.Errorf("holder %+v, record %+v", h, r)
|
|
}
|
|
p.Compose() // the pass the lease key's change wakes
|
|
if g, _, _ := p.Sealed(); !g.LeaseHanded {
|
|
t.Error("the user-list key does not say the lease is handed")
|
|
}
|
|
if _, err := p.HandBack("operator@shanks", "back"); err == nil || !strings.HasPrefix(err.Error(), "gone") {
|
|
t.Errorf("the user list's hand-back once the lease is handed: %v, want gone", err)
|
|
}
|
|
if _, err := p.LeaseWrite("operator@shanks", "again"); err == nil {
|
|
t.Error("a second lease-write with the lease handed was asked")
|
|
}
|
|
|
|
// 774/S18/AC1: the key or the expiry changed by another hand is said once, and nothing is written.
|
|
f.log = nil
|
|
f.put(grants.ReadLease, grants.LeaseKey, map[string]any{"instance": "controller@a pid 9", "host": "box-a", "epoch": 61})
|
|
p.Compose()
|
|
p.Compose()
|
|
f.expiry = 15 * time.Second
|
|
p.Compose()
|
|
if !slices.Equal(f.log, []string{"event lease-altered"}) || !strings.Contains(record(t, p).Altered, "controller@a pid 9") {
|
|
t.Errorf("an altered lease said %v (%q), want lease-altered once naming it", f.log, record(t, p).Altered)
|
|
}
|
|
})
|
|
|
|
// Issue 774 (the lease write met at novox): the controller's service is stopped on the node placement places
|
|
// mesh-controller on, never on the lease's host (a hostname); none or two such nodes refuse by name, and a failed act
|
|
// is logged beside its condition.
|
|
t.Run("the controller's node", func(t *testing.T) {
|
|
for _, row := range []struct {
|
|
name, stop, said string
|
|
place func(f *leaseFake)
|
|
}{
|
|
{"placed on a", "a", "", func(*leaseFake) {}},
|
|
{"placed nowhere", "", "no node places mesh-controller", func(f *leaseFake) {
|
|
delete(f.states[grants.ReadPlacement], "placement.a")
|
|
f.assign("a", "nats")
|
|
f.assign("a", "mod-x")
|
|
}},
|
|
{"placed twice", "", "two or more nodes place mesh-controller (a, b)", func(f *leaseFake) { f.assign("b", grants.Controller) }},
|
|
} {
|
|
f, p := newLeaseFake(t)
|
|
if _, err := p.LeaseWrite("operator@shanks", "the controller goes"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
p.Compose()
|
|
row.place(f)
|
|
if err := warrant(t, p, "yes"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
stops := slices.DeleteFunc(slices.Clone(f.calls), func(c string) bool { return !strings.HasPrefix(c, grants.CallServiceStop+"@") })
|
|
r := record(t, p)
|
|
switch {
|
|
case row.stop != "" && (!slices.Equal(stops, []string{grants.CallServiceStop + "@" + row.stop}) || r.Node != row.stop):
|
|
t.Errorf("%s: stopped %v, node %q; want the stop on %s alone", row.name, stops, r.Node, row.stop)
|
|
case row.stop == "" && (len(stops) != 0 || r.Open != nil || len(r.Acts) != 1 || r.Acts[0].Step != grants.StepFailed ||
|
|
!strings.Contains(r.Acts[0].Said, row.said)):
|
|
t.Errorf("%s: stopped %v, %+v; want the act failed saying %q, nothing stopped", row.name, stops, r, row.said)
|
|
case row.stop == "" && !slices.ContainsFunc(f.logged, func(l string) bool { return strings.Contains(l, row.said) }):
|
|
t.Errorf("%s: logged %q, want a line saying %q", row.name, f.logged, row.said)
|
|
}
|
|
}
|
|
})
|
|
|
|
// 774/S19/AC1, AC2: hand-back restores the expiry, deletes holder and removal-due, and starts the controller only once
|
|
// the list with its rights is loaded; once the removal ran it is refused.
|
|
t.Run("hand-back", func(t *testing.T) {
|
|
f, p := handed(t)
|
|
if _, err := p.LeaseHandBack("operator@shanks", "it goes back"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
p.Compose()
|
|
f.log = nil
|
|
if err := warrant(t, p, "yes"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r := record(t, p)
|
|
if f.expiry != 15*time.Second || f.states[grants.ReadLease][grants.LeaseKey] != nil || r.Cut || f.active != "inactive" {
|
|
t.Fatalf("after the warrant: expiry %s, holder %s, cut %v, controller %s", f.expiry, f.states[grants.ReadLease][grants.LeaseKey], r.Cut, f.active)
|
|
}
|
|
if _, _, found, _ := f.ownGet(grants.StateLease, grants.KeyRemovalDue); found {
|
|
t.Error("removal-due is not deleted")
|
|
}
|
|
f.loads(p)
|
|
p.Compose()
|
|
if r := record(t, p); f.active != "active" || r.Open != nil || r.Acts[len(r.Acts)-1].Step != grants.StepDone {
|
|
t.Errorf("once loaded: controller %s, %+v", f.active, r)
|
|
}
|
|
// AC2: once the removal ran, refused naming why.
|
|
f, p = handed(t)
|
|
p.RemovalDueHeard()
|
|
p.Compose()
|
|
if _, err := p.LeaseHandBack("operator@shanks", "too late"); err == nil || !strings.Contains(err.Error(), "without its consumer") {
|
|
t.Errorf("hand-back after the removal: %v", err)
|
|
}
|
|
})
|
|
|
|
// 774/S20/AC1, AC2, AC3: the marker of removal-due starts the removal; a failure says removal-failed and writes the key
|
|
// again with a day; a hand-back's delete removes nothing.
|
|
t.Run("the day-later removal", func(t *testing.T) {
|
|
f, p := handed(t)
|
|
f.log = nil
|
|
p.RemovalDueHeard()
|
|
p.Compose()
|
|
want := []string{"consumer removed", "hand-act removed the controller's consumer of DEAD_LETTER_NOTICES", "stream removed",
|
|
"hand-act removed the stream ASSIGNMENTS"}
|
|
if !slices.Equal(f.log, want) || record(t, p).Removed.IsZero() {
|
|
t.Errorf("the removal %v (removed %v), want %v", f.log, record(t, p).Removed, want)
|
|
}
|
|
|
|
f, p = handed(t)
|
|
f.log = nil
|
|
f.answer[grants.CallStreamRemove] = func(map[string]any) (any, error) {
|
|
return nil, errors.New("ASSIGNMENTS has 1 consumers: a reader is attached, so it was not removed")
|
|
}
|
|
p.RemovalDueHeard()
|
|
p.Compose()
|
|
if !slices.Contains(f.log, "event removal-failed") || f.ttls[grants.StateLease+"."+grants.KeyRemovalDue] != grants.RemovalAfter ||
|
|
!record(t, p).Removed.IsZero() {
|
|
t.Errorf("a failed removal: %v, removal-due %s", f.log, f.ttls[grants.StateLease+"."+grants.KeyRemovalDue])
|
|
}
|
|
|
|
f, p = handed(t)
|
|
if _, err := p.LeaseHandBack("operator@shanks", "it goes back"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
p.Compose()
|
|
if err := warrant(t, p, "yes"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
f.log = nil
|
|
p.RemovalDueHeard() // the hand-back's own delete of removal-due, heard
|
|
p.Compose()
|
|
if slices.ContainsFunc(f.log, func(s string) bool { return strings.Contains(s, "removed") }) {
|
|
t.Errorf("a hand-back's delete of removal-due removed %v", f.log)
|
|
}
|
|
})
|
|
|
|
// 774/S21/AC1: once the lease is handed away a membership is written to the state and not published on its subject.
|
|
t.Run("memberships after the write", func(t *testing.T) {
|
|
f, p := handed(t)
|
|
f.publishes = nil
|
|
f.assign("b", "mod-x")
|
|
p.Compose()
|
|
if len(f.publishes) != 0 {
|
|
t.Errorf("published on mesh.assignment after the lease write: %v", f.publishes)
|
|
}
|
|
if _, _, found, _ := f.ownGet(grants.StateMemberships, "b.mod-x"); !found {
|
|
t.Error("the membership is not written to the state")
|
|
}
|
|
})
|
|
}
|
|
|
|
// handed is a part whose lease write is done: the lease names mesh-grants and removal-due is written.
|
|
func handed(t *testing.T) (*leaseFake, *grants.Part) {
|
|
t.Helper()
|
|
f, p := newLeaseFake(t)
|
|
if _, err := p.LeaseWrite("operator@shanks", "the controller goes"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
p.Compose()
|
|
if err := warrant(t, p, "yes"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
f.loads(p)
|
|
p.Compose()
|
|
if r := record(t, p); r.Handed.IsZero() {
|
|
t.Fatalf("the write did not finish: %+v, %v", r, f.log)
|
|
}
|
|
return f, p
|
|
}
|