mesh-delivery/modules/mesh-grants/leasewrite_test.go

package manifest

import (
	"encoding/json"
	"errors"
	"fmt"
	"slices"
	"strings"
	"testing"
	"time"

	"git.novox.be/novox/mesh-sdk/go/asks"
	"git.novox.be/novox/mesh-sdk/go/buscompose"
	"git.novox.be/novox/mesh-sdk/go/delivery"
	"git.novox.be/novox/mesh-sdk/go/health"
	"git.novox.be/novox/mesh-sdk/go/seal"

	"mesh-grants/internal/grants"
)

// leaseFake is the bus as the lease write meets it: fakeBus's states and tools, and the seats and tools the write asks.
// Each answer is shaped as its writer's code answers it (issue 774): node-service-manager's status the systemd module's
// Manager.Status (ActiveState), nats_bucket_expiry nats-tools' BucketExpirySet (now), nats_consumer_remove its
// ConsumerRemoved (removed) and an absent consumer its error ("has no consumer"), nats_stream_remove its StreamRemoved
// (removed, present), issue-tracker.get mesh-issues' Brief (status), node_trusts mesh-delivery-view's TrustsAnswer
// (every), mesh-config's compare its Comparison (source, count), mesh-wants' upgrade compare the assigner's answer
// (empty), vault_status mesh-vault's answer (ready, minting.on, minting.held_back).
type leaseFake struct {
	*fakeBus
	answer map[string]func(map[string]any) (any, error)
	active string        // the controller's ActiveState
	expiry time.Duration // the lease bucket's
	ttls   map[string]time.Duration
	log    []string // the write's acts on the bus, in order
}

func newLeaseFake(t *testing.T) (*leaseFake, *grants.Part) {
	t.Helper()
	b := fixture()
	b.manifestOf("nats", `{"module":"nats","version":"1","claims":[{"name":"mesh-broker"}],"own-secrets":{"broker":"/var/lib/nats/broker"},`+
		`"tools":["nats_bucket_expiry","nats_consumer_remove","nats_stream_remove"]}`)
	b.put(grants.ReadLease, grants.LeaseKey, map[string]any{"instance": "controller@a pid 1", "host": "a", "epoch": 60,
		"taken": "2026-10-30T14:52:10Z"})
	b.put(grants.ReadTakeover, health.TakeoverKey, health.Takeover{V: 1, Follow: health.FollowKeeper, By: "operator", Why: "the keeper took over"})
	busKey, err := seal.GenerateKeyPair()
	if err != nil {
		t.Fatal(err)
	}
	b.put(grants.ReadHave, "a.nats", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK,
		UserList: &delivery.UserListWritten{Source: "a.nats.src2", Revision: 7, Digest: "sha256:aa", Users: 6, Key: busKey.PublicKey}})
	// The controller's hash in the vault, so the list carries the controller and the cut changes it.
	b.put(grants.ReadAccounts, buscompose.ControllerName, map[string]any{"hash": "$2a$11$controller", "generation": 1,
		"at": "2026-10-30T14:52:10Z"})
	f := &leaseFake{fakeBus: b, active: "active", expiry: 15 * time.Second, ttls: map[string]time.Duration{}}
	f.answer = map[string]func(map[string]any) (any, error){
		grants.CallIssue:         func(map[string]any) (any, error) { return map[string]any{"status": "resolved"}, nil },
		grants.CallNodeTrusts:    func(map[string]any) (any, error) { return map[string]any{"every": true, "not_yet": []string{}}, nil },
		grants.CallConfigCompare: func(map[string]any) (any, error) { return map[string]any{"source": "indexer", "count": 0}, nil },
		grants.CallUpgrade:       func(map[string]any) (any, error) { return map[string]any{"empty": true, "splits": []any{}}, nil },
		grants.CallVaultStatus: func(map[string]any) (any, error) {
			return map[string]any{"ready": true, "minting": map[string]any{"on": true, "held_back": map[string]string{}}}, nil
		},
		grants.CallServiceStop:   func(map[string]any) (any, error) { f.active = "inactive"; return map[string]any{"ok": true}, nil },
		grants.CallServiceStart:  func(map[string]any) (any, error) { f.active = "active"; return map[string]any{"ok": true}, nil },
		grants.CallServiceStatus: func(map[string]any) (any, error) { return map[string]any{"ActiveState": f.active}, nil },
		grants.CallBucketExpiry: func(a map[string]any) (any, error) {
			now := fmt.Sprint(a["max_age"])
			d, _ := time.ParseDuration(strings.Replace(now, "none", "0s", 1))
			f.expiry = d
			f.log = append(f.log, "expiry "+now)
			return map[string]any{"bucket": a["bucket"], "now": now, "changed": true}, nil
		},
		grants.CallRecordWarrant: func(a map[string]any) (any, error) { f.log = append(f.log, "warranted"); return map[string]any{}, nil },
		grants.CallRecordHandAct: func(a map[string]any) (any, error) {
			f.log = append(f.log, "hand-act "+fmt.Sprint(a["what"]))
			return map[string]any{}, nil
		},
		grants.CallConsumerRemove: func(map[string]any) (any, error) {
			f.log = append(f.log, "consumer removed")
			return map[string]any{"removed": true}, nil
		},
		grants.CallStreamRemove: func(map[string]any) (any, error) {
			f.log = append(f.log, "stream removed")
			return map[string]any{"removed": true, "present": true}, nil
		},
	}
	p := partOn(b)
	p.Call, p.Store, p.Lease, p.Asker, p.Objects = f, ttlStore{ownStore{b}, f}, f, &fakeAsker{wheres: map[string]*asks.Warrant{}}, &fakeObjects{}
	p.SaySeat = func(event string, _ any) error { f.log = append(f.log, "event "+event); return nil }
	p.Nonce = func() string { return "0001" }
	return f, p
}

func (f *leaseFake) Call(key string, body any) (json.RawMessage, error) {
	base, _, _ := strings.Cut(key, "@")
	if answer, ok := f.answer[base]; ok {
		f.mu.Lock()
		f.calls = append(f.calls, key)
		f.mu.Unlock()
		args, _ := body.(map[string]any)
		out, err := answer(args)
		if err != nil {
			return nil, err
		}
		return json.Marshal(out)
	}
	return f.fakeBus.Call(key, body)
}

func (f *leaseFake) PutHolder(value []byte) error {
	f.log = append(f.log, "holder")
	f.states[grants.ReadLease][grants.LeaseKey] = value
	return nil
}

func (f *leaseFake) DeleteHolder() error {
	f.log = append(f.log, "holder deleted")
	delete(f.states[grants.ReadLease], grants.LeaseKey)
	return nil
}

func (f *leaseFake) Expiry() (time.Duration, error) { return f.expiry, nil }

// loads says the bus node loaded the list the part last sealed.
func (f *leaseFake) loads(p *grants.Part) {
	g, _, _ := p.Sealed()
	f.put(grants.ReadHave, "a.nats", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK,
		UserList: &delivery.UserListWritten{Source: delivery.GrantsUserListSource, Revision: g.Revision, Digest: g.Digest,
			Users: g.Users, Key: keyOf(f)}})
}

func keyOf(f *leaseFake) string {
	var h delivery.Have
	_ = json.Unmarshal(f.states[grants.ReadHave]["a.nats"], &h)
	return h.UserList.Key
}

// ttlStore is the part's own store with a time to live per key, recorded.
type ttlStore struct {
	ownStore
	f *leaseFake
}

func (s ttlStore) PutTTL(state, key string, value []byte, ttl time.Duration) (uint64, error) {
	s.f.ttls[state+"."+key] = ttl
	s.f.log = append(s.f.log, key+" "+ttl.String())
	return s.Put(state, key, value)
}

// record is the part's lease record now.
func record(t *testing.T, p *grants.Part) grants.LeaseRecord {
	t.Helper()
	r, err := p.LeaseState()
	if err != nil {
		t.Fatal(err)
	}
	return r
}

// warrant answers the open act's ask with option, as the router's warrant hands it.
func warrant(t *testing.T, p *grants.Part, option string) error {
	t.Helper()
	r := record(t, p)
	if r.Open == nil {
		t.Fatal("no act is open")
	}
	return p.Warranted(asks.Option{ID: option}, asks.Warrant{Ask: r.Open.Ask, Option: option, Outcome: asks.OutcomeChosen})
}

// Issue 774 (novox/hq ADR 0582): the lease write, from its ask through the day-later removal, against stand-in seats.
func TestTheLeaseWrite(t *testing.T) {
	// 774/S14/AC1: any one precondition not green refuses the verb naming it; all green asks.
	t.Run("preconditions", func(t *testing.T) {
		for _, row := range []struct {
			name   string
			break_ func(f *leaseFake)
		}{
			{"lease-check", func(f *leaseFake) {
				f.manifestOf("mod-u", `{"module":"mod-u","version":"1","reads":["mesh-controller.conditions"]}`)
			}},
			{"earlier-steps", func(f *leaseFake) {
				f.answer[grants.CallIssue] = func(a map[string]any) (any, error) {
					if fmt.Sprint(a["number"]) == "1954" {
						return map[string]any{"status": "fixing"}, nil
					}
					return map[string]any{"status": "resolved"}, nil
				}
			}},
			{"node-trusts", func(f *leaseFake) {
				f.answer[grants.CallNodeTrusts] = func(map[string]any) (any, error) {
					return map[string]any{"every": false, "not_yet": []string{"ace"}}, nil
				}
			}},
			{"config-compare", func(f *leaseFake) {
				f.answer[grants.CallConfigCompare] = func(map[string]any) (any, error) { return map[string]any{"source": "indexer", "count": 2}, nil }
			}},
			{"upgrade-compare", func(f *leaseFake) {
				f.answer[grants.CallUpgrade] = func(map[string]any) (any, error) { return map[string]any{"empty": false, "splits": []any{1}}, nil }
			}},
			{"keeper-take-over", func(f *leaseFake) {
				f.put(grants.ReadTakeover, health.TakeoverKey, health.Takeover{V: 1, Follow: health.FollowController})
			}},
			{"vault-minting", func(f *leaseFake) {
				f.answer[grants.CallVaultStatus] = func(map[string]any) (any, error) {
					return map[string]any{"ready": true, "minting": map[string]any{"on": true, "held_back": map[string]string{"a.x": "why"}}}, nil
				}
			}},
			{"bus-tools", func(f *leaseFake) { f.manifestOf("nats", natsManifest) }},
			{"", func(*leaseFake) {}},
		} {
			f, p := newLeaseFake(t)
			row.break_(f)
			_, err := p.LeaseWrite("operator@shanks", "the controller goes")
			switch {
			case row.name == "" && err != nil:
				t.Errorf("all green: refused: %v", err)
			case row.name != "" && (err == nil || !strings.Contains(err.Error(), row.name+": ")):
				t.Errorf("%s not green: %v, want a refusal naming it", row.name, err)
			case row.name != "":
				for _, other := range grants.LeaseCheckNames {
					if other != row.name && strings.Contains(err.Error(), other+": ") {
						t.Errorf("%s not green: the refusal names %s too: %v", row.name, other, err)
					}
				}
			}
			if r := record(t, p); (r.Open != nil) != (row.name == "") {
				t.Errorf("%q: an act open %v", row.name, r.Open)
			}
		}
	})

	// 774/S15/AC1, AC2: a no, an expiry, or a warrant whose checks changed writes nothing, naming the changed check.
	t.Run("the ask and the warrant", func(t *testing.T) {
		for _, row := range []struct {
			name, step, said string
			answer           func(t *testing.T, f *leaseFake, p *grants.Part)
		}{
			{"declined", grants.StepDeclined, "said no", func(t *testing.T, _ *leaseFake, p *grants.Part) {
				if err := warrant(t, p, "no"); err != nil {
					t.Fatal(err)
				}
			}},
			{"expired", "expired", "expired", func(t *testing.T, _ *leaseFake, p *grants.Part) {
				r := record(t, p)
				p.Said("", asks.Warrant{Ask: r.Open.Ask, Outcome: asks.OutcomeExpired}, nil)
			}},
			{"checks changed", grants.StepRefused, "the checks changed since the ask: vault-minting (now: ready true, minting true, 1 held back)",
				func(t *testing.T, f *leaseFake, p *grants.Part) {
					// The vault holds one account back between the ask and the warrant.
					f.answer[grants.CallVaultStatus] = func(map[string]any) (any, error) {
						return map[string]any{"ready": true, "minting": map[string]any{"on": true, "held_back": map[string]string{"a.x": "why"}}}, nil
					}
					if err := warrant(t, p, "yes"); err != nil {
						t.Fatal(err)
					}
				}},
		} {
			f, p := newLeaseFake(t)
			if _, err := p.LeaseWrite("operator@shanks", "the controller goes"); err != nil {
				t.Fatal(err)
			}
			p.Compose() // the process asks it
			if a := p.Asker.(*fakeAsker); len(a.asks) != 1 || len(a.asks[0].Options) != 2 || a.asks[0].Options[0].Binds == "" {
				t.Fatalf("%s: asked %+v, want one ask whose yes binds the act", row.name, a.asks)
			}
			row.answer(t, f, p)
			r := record(t, p)
			if r.Open != nil || len(r.Acts) != 1 || r.Acts[0].Step != row.step || !strings.Contains(r.Acts[0].Said, row.said) {
				t.Errorf("%s: open %v, acts %+v; want one act %s saying %q", row.name, r.Open, r.Acts, row.step, row.said)
			}
			if f.count(grants.CallServiceStop+"@a") != 0 || len(f.log) != 0 || r.Cut {
				t.Errorf("%s: something was written: stops %d, %v, cut %v", row.name, f.count(grants.CallServiceStop+"@a"), f.log, r.Cut)
			}
		}
	})

	// 774/S16/AC1, AC2; 774/S17/AC1, AC2: the write's six steps, in order, each read back before the next.
	t.Run("the write", func(t *testing.T) {
		f, p := newLeaseFake(t)
		f.answer[grants.CallServiceStop] = func(map[string]any) (any, error) { return map[string]any{"ok": true}, nil } // reads back active
		if _, err := p.LeaseWrite("operator@shanks", "the controller goes"); err != nil {
			t.Fatal(err)
		}
		p.Compose()
		if err := warrant(t, p, "yes"); err != nil {
			t.Fatal(err)
		}
		if r := record(t, p); r.Open != nil || r.Cut || r.Acts[0].Step != grants.StepFailed || !strings.Contains(r.Acts[0].Said, "did not read back stopped") || len(f.log) != 0 {
			t.Fatalf("a stop not read back: %+v, %v; want the act failed there, nothing written", r, f.log)
		}

		f, p = newLeaseFake(t)
		if _, err := p.LeaseWrite("operator@shanks", "the controller goes"); err != nil {
			t.Fatal(err)
		}
		p.Compose()
		before, _, _ := p.Sealed()
		if err := warrant(t, p, "yes"); err != nil {
			t.Fatal(err)
		}
		r := record(t, p)
		g, _, _ := p.Sealed()
		if f.active != "inactive" || !r.Cut || r.Open == nil || r.Open.Step != grants.StepCut || r.Open.List != g.Digest || g.Digest == before.Digest {
			t.Fatalf("after the warrant: active %s, %+v, sealed %s (was %s); want stopped, cut and waiting on the cut list", f.active, r, g.Digest, before.Digest)
		}
		if len(f.log) != 0 {
			t.Fatalf("the expiry or the key was touched before the cut list loaded: %v", f.log)
		}
		// The cut list takes the controller's write on the lease and gives mesh-grants the key holder.
		var st grants.UserListState
		raw, _, _, _ := ownStore{f.fakeBus}.Get(grants.StateUserList, grants.KeyList)
		_ = json.Unmarshal(raw, &st)
		p.Compose() // a pass with the bus node not loaded yet waits
		if len(f.log) != 0 {
			t.Fatalf("a pass before the load acted: %v", f.log)
		}
		f.loads(p)
		p.Compose()
		want := []string{"expiry none", "holder", "event lease-handed", "warranted", "removal-due 24h0m0s"}
		if !slices.Equal(f.log, want) {
			t.Errorf("the write's acts %v, want %v in that order", f.log, want)
		}
		var h buscompose.LeaseHolder
		_ = json.Unmarshal(f.states[grants.ReadLease][grants.LeaseKey], &h)
		r = record(t, p)
		if h.HandedTo != "mesh-grants" || h.HandedBy != "operator" || h.Why != "the controller goes" || h.Ask != r.Ask || h.At.IsZero() ||
			r.Open != nil || r.Handed.IsZero() || r.Acts[len(r.Acts)-1].Step != grants.StepDone {
			t.Errorf("holder %+v, record %+v", h, r)
		}
		p.Compose() // the pass the lease key's change wakes
		if g, _, _ := p.Sealed(); !g.LeaseHanded {
			t.Error("the user-list key does not say the lease is handed")
		}
		if _, err := p.HandBack("operator@shanks", "back"); err == nil || !strings.HasPrefix(err.Error(), "gone") {
			t.Errorf("the user list's hand-back once the lease is handed: %v, want gone", err)
		}
		if _, err := p.LeaseWrite("operator@shanks", "again"); err == nil {
			t.Error("a second lease-write with the lease handed was asked")
		}

		// 774/S18/AC1: the key or the expiry changed by another hand is said once, and nothing is written.
		f.log = nil
		f.put(grants.ReadLease, grants.LeaseKey, map[string]any{"instance": "controller@a pid 9", "host": "a", "epoch": 61})
		p.Compose()
		p.Compose()
		f.expiry = 15 * time.Second
		p.Compose()
		if !slices.Equal(f.log, []string{"event lease-altered"}) || !strings.Contains(record(t, p).Altered, "controller@a pid 9") {
			t.Errorf("an altered lease said %v (%q), want lease-altered once naming it", f.log, record(t, p).Altered)
		}
	})

	// 774/S19/AC1, AC2: hand-back restores the expiry, deletes holder and removal-due, and starts the controller only once
	// the list with its rights is loaded; once the removal ran it is refused.
	t.Run("hand-back", func(t *testing.T) {
		f, p := handed(t)
		if _, err := p.LeaseHandBack("operator@shanks", "it goes back"); err != nil {
			t.Fatal(err)
		}
		p.Compose()
		f.log = nil
		if err := warrant(t, p, "yes"); err != nil {
			t.Fatal(err)
		}
		r := record(t, p)
		if f.expiry != 15*time.Second || f.states[grants.ReadLease][grants.LeaseKey] != nil || r.Cut || f.active != "inactive" {
			t.Fatalf("after the warrant: expiry %s, holder %s, cut %v, controller %s", f.expiry, f.states[grants.ReadLease][grants.LeaseKey], r.Cut, f.active)
		}
		if _, _, found, _ := f.ownGet(grants.StateLease, grants.KeyRemovalDue); found {
			t.Error("removal-due is not deleted")
		}
		f.loads(p)
		p.Compose()
		if r := record(t, p); f.active != "active" || r.Open != nil || r.Acts[len(r.Acts)-1].Step != grants.StepDone {
			t.Errorf("once loaded: controller %s, %+v", f.active, r)
		}
		// AC2: once the removal ran, refused naming why.
		f, p = handed(t)
		p.RemovalDueHeard()
		p.Compose()
		if _, err := p.LeaseHandBack("operator@shanks", "too late"); err == nil || !strings.Contains(err.Error(), "without its consumer") {
			t.Errorf("hand-back after the removal: %v", err)
		}
	})

	// 774/S20/AC1, AC2, AC3: the marker of removal-due starts the removal; a failure says removal-failed and writes the key
	// again with a day; a hand-back's delete removes nothing.
	t.Run("the day-later removal", func(t *testing.T) {
		f, p := handed(t)
		f.log = nil
		p.RemovalDueHeard()
		p.Compose()
		want := []string{"consumer removed", "hand-act removed the controller's consumer of DEAD_LETTER_NOTICES", "stream removed",
			"hand-act removed the stream ASSIGNMENTS"}
		if !slices.Equal(f.log, want) || record(t, p).Removed.IsZero() {
			t.Errorf("the removal %v (removed %v), want %v", f.log, record(t, p).Removed, want)
		}

		f, p = handed(t)
		f.log = nil
		f.answer[grants.CallStreamRemove] = func(map[string]any) (any, error) {
			return nil, errors.New("ASSIGNMENTS has 1 consumers: a reader is attached, so it was not removed")
		}
		p.RemovalDueHeard()
		p.Compose()
		if !slices.Contains(f.log, "event removal-failed") || f.ttls[grants.StateLease+"."+grants.KeyRemovalDue] != grants.RemovalAfter ||
			!record(t, p).Removed.IsZero() {
			t.Errorf("a failed removal: %v, removal-due %s", f.log, f.ttls[grants.StateLease+"."+grants.KeyRemovalDue])
		}

		f, p = handed(t)
		if _, err := p.LeaseHandBack("operator@shanks", "it goes back"); err != nil {
			t.Fatal(err)
		}
		p.Compose()
		if err := warrant(t, p, "yes"); err != nil {
			t.Fatal(err)
		}
		f.log = nil
		p.RemovalDueHeard() // the hand-back's own delete of removal-due, heard
		p.Compose()
		if slices.ContainsFunc(f.log, func(s string) bool { return strings.Contains(s, "removed") }) {
			t.Errorf("a hand-back's delete of removal-due removed %v", f.log)
		}
	})

	// 774/S21/AC1: once the lease is handed away a membership is written to the state and not published on its subject.
	t.Run("memberships after the write", func(t *testing.T) {
		f, p := handed(t)
		f.publishes = nil
		f.assign("b", "mod-x")
		p.Compose()
		if len(f.publishes) != 0 {
			t.Errorf("published on mesh.assignment after the lease write: %v", f.publishes)
		}
		if _, _, found, _ := f.ownGet(grants.StateMemberships, "b.mod-x"); !found {
			t.Error("the membership is not written to the state")
		}
	})
}

// handed is a part whose lease write is done: the lease names mesh-grants and removal-due is written.
func handed(t *testing.T) (*leaseFake, *grants.Part) {
	t.Helper()
	f, p := newLeaseFake(t)
	if _, err := p.LeaseWrite("operator@shanks", "the controller goes"); err != nil {
		t.Fatal(err)
	}
	p.Compose()
	if err := warrant(t, p, "yes"); err != nil {
		t.Fatal(err)
	}
	f.loads(p)
	p.Compose()
	if r := record(t, p); r.Handed.IsZero() {
		t.Fatalf("the write did not finish: %+v, %v", r, f.log)
	}
	return f, p
}