514 lines
20 KiB
Go
514 lines
20 KiB
Go
package userlist
|
|
|
|
import (
|
|
"context"
|
|
"crypto/ed25519"
|
|
"encoding/json"
|
|
"errors"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
"git.novox.be/novox/mesh-sdk/go/seal"
|
|
"github.com/novox/mesh-host/internal/agent"
|
|
)
|
|
|
|
const node = "novox"
|
|
|
|
func key() (ed25519.PublicKey, ed25519.PrivateKey) {
|
|
pub, k, err := ed25519.GenerateKey(nil)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
return pub, k
|
|
}
|
|
|
|
var controllerPub, controllerKey = key()
|
|
var _, strangerKey = key()
|
|
|
|
func fpOf(c string) string { return delivery.SourcePrefix2 + strings.Repeat(c, 64) }
|
|
|
|
// feed is a watch the test writes to: the keys there before the watch, Caught, then each put. watching is closed once
|
|
// the watch began, so a put after it is never handed before the keys there before it.
|
|
type feed struct {
|
|
ch chan agent.Update
|
|
rev uint64
|
|
first []agent.Update
|
|
// watching is closed once the watch caught up, so a later put is not read before the keys there before it.
|
|
watching chan struct{}
|
|
once sync.Once
|
|
// err refuses the watch, as the bus refuses a watch it grants nothing of.
|
|
err error
|
|
}
|
|
|
|
func (f *feed) Watch(context.Context, string) (<-chan agent.Update, error) {
|
|
if f.err != nil {
|
|
return nil, f.err
|
|
}
|
|
for _, u := range f.first {
|
|
f.ch <- u
|
|
}
|
|
f.ch <- agent.Update{Caught: true}
|
|
f.once.Do(func() {
|
|
if f.watching != nil {
|
|
close(f.watching)
|
|
}
|
|
})
|
|
return f.ch, nil
|
|
}
|
|
|
|
// hashed is a list whose users carry a hash of bcrypt's form, as the composer writes them.
|
|
func hashed(users ...string) string {
|
|
var b strings.Builder
|
|
b.WriteString("authorization {\n users = [\n")
|
|
for _, u := range users {
|
|
b.WriteString(` { user: "` + u + `", password: "$2a$11$` + strings.Repeat("h", 53) + `", permissions: {` + "\n")
|
|
}
|
|
b.WriteString(" ]\n}\n")
|
|
return b.String()
|
|
}
|
|
|
|
var vault, _ = seal.GenerateKeyPair()
|
|
|
|
func list(users ...string) string {
|
|
var b strings.Builder
|
|
b.WriteString("authorization {\n users = [\n")
|
|
for _, u := range users {
|
|
b.WriteString(` {user: "` + u + `", password: "$2a$11$x"}` + "\n")
|
|
}
|
|
b.WriteString(" ]\n}\n")
|
|
return b.String()
|
|
}
|
|
|
|
// comp is a composition of module at fp holding the user list content (none when empty), signed by signer; by digest
|
|
// when byDigest, as the controller sends a file over 64 KiB; for another node or module than its key when asked.
|
|
func comp(t *testing.T, module, fp, content string, signer ed25519.PrivateKey, p put) (string, []byte) {
|
|
t.Helper()
|
|
owner := module
|
|
if p.asModule != "" {
|
|
owner = p.asModule
|
|
}
|
|
server, _ := json.Marshal(map[string]any{"id": owner + ".server-conf", "type": "file", "module": owner,
|
|
"path": "/var/lib/nats-module/conf/nats.conf", "content": "include accounts.conf\n"})
|
|
rs := []json.RawMessage{server}
|
|
if content != "" {
|
|
r := map[string]any{"id": owner + ".bus-users", "type": "file", "module": owner,
|
|
"path": "/var/lib/nats-module/conf/accounts.conf", "content": content, "mode": "0600"}
|
|
if p.byDigest {
|
|
delete(r, "content")
|
|
r["digest"], r["size"] = Digest([]byte(content)), len(content)
|
|
}
|
|
users, _ := json.Marshal(r)
|
|
rs = append(rs, users)
|
|
}
|
|
c := delivery.Composed{Node: node, Module: module, FP: fp, Resources: rs}
|
|
if p.asNode != "" {
|
|
c.Node = p.asNode
|
|
}
|
|
if p.asModule != "" {
|
|
c.Module = p.asModule
|
|
}
|
|
k, err := delivery.ComposedKey(node, module, fp)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
signed, err := delivery.SignComposed(c, signer)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
raw, err := delivery.EncodeComposed(signed)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return k, raw
|
|
}
|
|
|
|
// gput is the grants part's key as the bus node hears it at the watch's start (ADR 0547 clauses 7, 8): which list to
|
|
// follow, and the list sealed to the node's key; one sealed to another key, or one that does not open to the key's digest.
|
|
type gput struct {
|
|
follow, content string
|
|
otherKey, badDigest bool
|
|
// after puts it once the grants part's list was written, as a hand-back comes after a take-over.
|
|
after bool
|
|
}
|
|
|
|
type put struct {
|
|
module, fp, content string
|
|
stranger, deleted bool
|
|
byDigest bool
|
|
asNode, asModule string
|
|
}
|
|
|
|
// Defends novox/hq ADR 0547 §8: the bus node writes the newest signed list alone, only when it changed, and reports it.
|
|
func TestTheBusNodeWritesTheNewestSignedUserListAlone(t *testing.T) {
|
|
one, two, three := list("controller", "a"), list("controller", "a", "b"), list("controller", "a", "b", "c")
|
|
five := list("controller", "a", "b", "c", "d")
|
|
for _, c := range []struct {
|
|
name string
|
|
before, after []put
|
|
again bool
|
|
failWrite bool
|
|
onDisk string
|
|
wantWrites []string // contents written, in order
|
|
wantUsers int
|
|
wantErr string
|
|
wantRefused bool
|
|
grants []gput
|
|
// grantsRefused refuses the watch of the grants part's key; kept is the key last heard, kept on the machine;
|
|
// wantUnwatched says the have then says why it is not watched.
|
|
grantsRefused bool
|
|
kept *gput
|
|
wantUnwatched bool
|
|
// wantSource and wantFollows are what the last have says it took the list from and follows.
|
|
wantSource, wantFollows string
|
|
// vault gives the follower the vault's key; wantHanded is how many users it then hands from the list on the
|
|
// machine (ADR 0547 clause 9).
|
|
vault bool
|
|
wantHanded int
|
|
}{
|
|
{name: "the list there is written once", before: []put{{module: "nats", fp: fpOf("a"), content: two}},
|
|
wantWrites: []string{two}, wantUsers: 3},
|
|
{name: "a newer revision is written, the older fingerprint is not",
|
|
before: []put{{module: "nats", fp: fpOf("a"), content: one}, {module: "nats", fp: fpOf("b"), content: two}},
|
|
after: []put{{module: "nats", fp: fpOf("a"), content: three}},
|
|
wantWrites: []string{two, three}, wantUsers: 4},
|
|
{name: "a composition signed by a stranger is no source",
|
|
before: []put{{module: "nats", fp: fpOf("a"), content: one}},
|
|
after: []put{{module: "nats", fp: fpOf("b"), content: three, stranger: true}},
|
|
wantWrites: []string{one}, wantUsers: 2},
|
|
{name: "a composition without the list is ignored", before: []put{{module: "web", fp: fpOf("a"), content: ""}}},
|
|
{name: "a deleted key is no source",
|
|
before: []put{{module: "nats", fp: fpOf("a"), content: one}, {module: "nats", fp: fpOf("b"), content: two}},
|
|
after: []put{{module: "nats", fp: fpOf("b"), content: "", deleted: true}, {module: "nats", fp: fpOf("a"), content: three}},
|
|
wantWrites: []string{two, three}, wantUsers: 4},
|
|
{name: "asked again, the same source is written again", before: []put{{module: "nats", fp: fpOf("a"), content: two}},
|
|
again: true, wantWrites: []string{two, two}, wantUsers: 3},
|
|
{name: "a list sent by digest counts the users of what it names",
|
|
before: []put{{module: "nats", fp: fpOf("a"), content: three, byDigest: true}},
|
|
wantWrites: []string{three}, wantUsers: 4},
|
|
{name: "another node's composition under this node's key is no source",
|
|
before: []put{{module: "nats", fp: fpOf("a"), content: two, asNode: "ace"}}},
|
|
{name: "another module's composition under the bus module's key is no source",
|
|
before: []put{{module: "nats", fp: fpOf("a"), content: two, asModule: "web"}}},
|
|
{name: "a list without the controller is refused and the file kept",
|
|
before: []put{{module: "nats", fp: fpOf("a"), content: list("a", "b")}},
|
|
onDisk: two, wantUsers: 2, wantErr: "holds no user \"controller\"", wantRefused: true},
|
|
{name: "a list that drops more than half is refused and the file kept",
|
|
before: []put{{module: "nats", fp: fpOf("a"), content: one}},
|
|
onDisk: five, wantUsers: 2, wantErr: "drops more than half", wantRefused: true},
|
|
{name: "a list that drops half is written", before: []put{{module: "nats", fp: fpOf("a"), content: two}},
|
|
onDisk: list("controller", "a", "b", "c", "d", "e"), wantWrites: []string{two}, wantUsers: 3},
|
|
{name: "the loaded list's hashes are handed sealed to the vault's key",
|
|
before: []put{{module: "nats", fp: fpOf("a"), content: hashed("controller", "node.ace")}}, vault: true,
|
|
wantWrites: []string{hashed("controller", "node.ace")}, wantUsers: 2, wantHanded: 2},
|
|
{name: "a refused list hands the list on the machine",
|
|
before: []put{{module: "nats", fp: fpOf("a"), content: list("a", "b")}}, vault: true,
|
|
onDisk: hashed("controller", "a", "b"), wantUsers: 2, wantErr: "holds no user", wantRefused: true, wantHanded: 3},
|
|
{name: "a failed write is said and kept", before: []put{{module: "nats", fp: fpOf("a"), content: two}}, failWrite: true,
|
|
wantWrites: []string{two}, wantUsers: 3, wantErr: "disk full"},
|
|
// ADR 0547 clauses 7, 8: never both lists; the grants part's from take-over, the controller's otherwise.
|
|
{name: "from take-over the grants part's list is written into the same file",
|
|
before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grants: []gput{{follow: delivery.FollowGrants, content: three}},
|
|
wantWrites: []string{three}, wantUsers: 4, wantSource: delivery.GrantsUserListSource, wantFollows: delivery.FollowGrants},
|
|
{name: "a hand-back writes the newest controller composition once",
|
|
before: []put{{module: "nats", fp: fpOf("a"), content: two}},
|
|
grants: []gput{{follow: delivery.FollowGrants, content: three}, {follow: delivery.FollowController, content: three, after: true}},
|
|
wantWrites: []string{three, two}, wantUsers: 3, wantSource: "novox.nats." + fpOf("a")[len(delivery.SourcePrefix2):], wantFollows: delivery.FollowController},
|
|
{name: "a refused watch with no key kept follows the controller's list and says it is not watched",
|
|
before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grantsRefused: true, wantUnwatched: true,
|
|
wantWrites: []string{two}, wantUsers: 3, wantSource: "novox.nats." + fpOf("a")[len(delivery.SourcePrefix2):], wantFollows: delivery.FollowController},
|
|
{name: "a refused watch keeps the follow last heard, never the controller's list",
|
|
before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grantsRefused: true, wantUnwatched: true,
|
|
kept: &gput{follow: delivery.FollowGrants, content: three},
|
|
wantWrites: []string{three}, wantUsers: 4, wantSource: delivery.GrantsUserListSource, wantFollows: delivery.FollowGrants},
|
|
{name: "in shadow the grants part's list is not taken",
|
|
before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grants: []gput{{follow: delivery.FollowController, content: three}},
|
|
wantWrites: []string{two}, wantUsers: 3, wantSource: "novox.nats." + fpOf("a")[len(delivery.SourcePrefix2):], wantFollows: delivery.FollowController},
|
|
{name: "a grants list that does not open to its digest is refused and the file kept",
|
|
before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grants: []gput{{follow: delivery.FollowGrants, content: three, badDigest: true}},
|
|
onDisk: two, wantErr: "could not be had", wantRefused: true, wantSource: delivery.GrantsUserListSource},
|
|
{name: "a grants list sealed to another key is refused and the file kept",
|
|
before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grants: []gput{{follow: delivery.FollowGrants, content: three, otherKey: true}},
|
|
onDisk: two, wantErr: "sealed to another key", wantRefused: true, wantSource: delivery.GrantsUserListSource},
|
|
} {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
f := &feed{ch: make(chan agent.Update, 64), watching: make(chan struct{})}
|
|
objects := map[string]string{}
|
|
var mu sync.Mutex
|
|
var writes []string
|
|
var reports []Written
|
|
reported := make(chan struct{}, 64)
|
|
gf := &feed{ch: make(chan agent.Update, 64)}
|
|
if c.grantsRefused {
|
|
gf.err = errors.New("Permissions Violation for Publish to $JS.API.STREAM.INFO.KV_mesh-grants_user-list")
|
|
}
|
|
var grantsAfter []agent.Update
|
|
var keptRaw []byte
|
|
sealed := map[string][]byte{}
|
|
nodeKey, err := seal.GenerateKeyPair()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
gs := c.grants
|
|
if c.kept != nil {
|
|
gs = append(append([]gput(nil), gs...), *c.kept)
|
|
}
|
|
for i, g := range gs {
|
|
to := nodeKey.PublicKey
|
|
if g.otherKey {
|
|
other, _ := seal.GenerateKeyPair()
|
|
to = other.PublicKey
|
|
}
|
|
digest := delivery.UserListDigest([]byte(g.content))
|
|
if g.badDigest {
|
|
digest = delivery.UserListDigest([]byte("another list"))
|
|
}
|
|
object, err := delivery.SealUserList([]byte(g.content), to)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sealed[delivery.UserListObjectName(digest)] = object
|
|
raw, err := delivery.EncodeGrantsUserList(delivery.GrantsUserList{Follow: g.follow, Digest: digest,
|
|
Object: delivery.UserListObjectName(digest), SealedTo: delivery.SealingKeyID(to), Node: node, Revision: uint64(i + 1)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
u := agent.Update{Key: delivery.GrantsUserListKey, Value: raw, Revision: uint64(i + 1)}
|
|
switch {
|
|
case c.kept != nil && i == len(gs)-1:
|
|
keptRaw = raw
|
|
case g.after:
|
|
grantsAfter = append(grantsAfter, u)
|
|
default:
|
|
gf.first = append(gf.first, u)
|
|
}
|
|
}
|
|
fl := New(Env{Node: node, Composed: f, Grants: gf,
|
|
Kept: func() ([]byte, uint64) { return keptRaw, 9 },
|
|
Keep: func([]byte, uint64) {},
|
|
Backoff: func(int) time.Duration { return time.Hour },
|
|
Fetch: func(_ context.Context, name string) ([]byte, error) {
|
|
if o, ok := sealed[name]; ok {
|
|
return o, nil
|
|
}
|
|
return nil, errors.New("no object " + name)
|
|
},
|
|
Open: func(b seal.Box) ([]byte, error) {
|
|
out, err := seal.Open(b, nodeKey.PrivateKey)
|
|
return []byte(out), err
|
|
},
|
|
Key: func() string { return nodeKey.PublicKey },
|
|
Trusted: func() []ed25519.PublicKey { return []ed25519.PublicKey{controllerPub} },
|
|
Resolve: func(_ context.Context, r json.RawMessage) ([]byte, error) {
|
|
var res struct{ Content, Digest string }
|
|
_ = json.Unmarshal(r, &res)
|
|
if res.Digest != "" {
|
|
body, ok := objects[res.Digest]
|
|
if !ok {
|
|
return nil, errors.New("no object " + res.Digest)
|
|
}
|
|
return []byte(body), nil
|
|
}
|
|
return []byte(res.Content), nil
|
|
},
|
|
OnDisk: func(json.RawMessage) ([]byte, error) {
|
|
if c.onDisk == "" {
|
|
return nil, nil
|
|
}
|
|
return []byte(c.onDisk), nil
|
|
},
|
|
Write: func(_ context.Context, module string, r json.RawMessage, body []byte) (string, bool, error) {
|
|
mu.Lock()
|
|
writes = append(writes, string(body))
|
|
mu.Unlock()
|
|
if c.failWrite {
|
|
return "", false, errors.New("disk full")
|
|
}
|
|
return Digest(body), true, nil
|
|
},
|
|
Report: func(_ context.Context, module string, w Written) {
|
|
mu.Lock()
|
|
reports = append(reports, w)
|
|
mu.Unlock()
|
|
reported <- struct{}{}
|
|
},
|
|
Now: func() time.Time { return time.Date(2026, 10, 25, 21, 50, 0, 0, time.UTC) },
|
|
VaultKey: func() string {
|
|
if c.vault {
|
|
return vault.PublicKey
|
|
}
|
|
return ""
|
|
},
|
|
})
|
|
update := func(p put) agent.Update {
|
|
f.rev++
|
|
signer := controllerKey
|
|
if p.stranger {
|
|
signer = strangerKey
|
|
}
|
|
k, raw := comp(t, p.module, p.fp, p.content, signer, p)
|
|
if p.byDigest {
|
|
objects[Digest([]byte(p.content))] = p.content
|
|
}
|
|
if p.deleted {
|
|
return agent.Update{Key: k, Revision: f.rev, Deleted: true}
|
|
}
|
|
return agent.Update{Key: k, Value: raw, Revision: f.rev}
|
|
}
|
|
for _, p := range c.before {
|
|
f.first = append(f.first, update(p))
|
|
}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
done := make(chan error, 1)
|
|
go func() { done <- fl.Run(ctx) }()
|
|
<-f.watching
|
|
// A later put after the first list is written: the first write waits for the grants part's key to catch up.
|
|
if len(c.after) > 0 {
|
|
for until := time.Now().Add(5 * time.Second); time.Now().Before(until); time.Sleep(5 * time.Millisecond) {
|
|
mu.Lock()
|
|
n := len(writes)
|
|
mu.Unlock()
|
|
if n > 0 {
|
|
break
|
|
}
|
|
}
|
|
}
|
|
for _, p := range c.after {
|
|
f.ch <- update(p)
|
|
}
|
|
if len(grantsAfter) > 0 {
|
|
for until := time.Now().Add(5 * time.Second); ; time.Sleep(5 * time.Millisecond) {
|
|
if _, st := fl.State(); st.Source == delivery.GrantsUserListSource || time.Now().After(until) {
|
|
break
|
|
}
|
|
}
|
|
for _, u := range grantsAfter {
|
|
gf.ch <- u
|
|
}
|
|
}
|
|
if c.again {
|
|
// After the first write is reported, so the look again is not folded into it.
|
|
select {
|
|
case <-reported:
|
|
case <-time.After(5 * time.Second):
|
|
}
|
|
fl.Again()
|
|
}
|
|
deadline := time.After(5 * time.Second)
|
|
for {
|
|
mu.Lock()
|
|
n := len(writes)
|
|
mu.Unlock()
|
|
if n >= len(c.wantWrites) {
|
|
break
|
|
}
|
|
select {
|
|
case <-deadline:
|
|
t.Fatalf("writes %d, want %d", n, len(c.wantWrites))
|
|
case <-time.After(5 * time.Millisecond):
|
|
}
|
|
}
|
|
if c.wantRefused {
|
|
select {
|
|
case <-reported:
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("a refused list was not reported")
|
|
}
|
|
}
|
|
if c.wantSource != "" {
|
|
for until := time.Now().Add(5 * time.Second); ; {
|
|
if _, st := fl.State(); st.Source == c.wantSource && (c.wantErr == "" || st.Error != "") || time.Now().After(until) {
|
|
break
|
|
}
|
|
time.Sleep(5 * time.Millisecond)
|
|
}
|
|
}
|
|
time.Sleep(50 * time.Millisecond) // nothing more is written
|
|
cancel()
|
|
<-done
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
if strings.Join(writes, "|") != strings.Join(c.wantWrites, "|") {
|
|
t.Fatalf("written %q, want %q", writes, c.wantWrites)
|
|
}
|
|
module, st := fl.State()
|
|
if c.wantUnwatched != (st.Unwatched != "") {
|
|
t.Fatalf("the have says unwatched %q", st.Unwatched)
|
|
}
|
|
// While the grants part's list is followed, an apply of the controller's composition writes not its list.
|
|
if _, raw := comp(t, "nats", fpOf("a"), two, controllerKey, put{}); c.wantFollows != "" {
|
|
cmp, err := delivery.DecodeComposed(raw)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
held, err := fl.Hold(cmp)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_, _, _, hasList := userList(held.Resources)
|
|
if want := c.wantFollows == delivery.FollowController; hasList != want || len(held.Resources) < 1 {
|
|
t.Fatalf("an apply while following %s holds the controller's list: %v", c.wantFollows, hasList)
|
|
}
|
|
// A list on the machine that cannot be read stops the apply; it is never left out to be removed.
|
|
if c.wantFollows == delivery.FollowGrants {
|
|
readErr := New(Env{Node: node, Composed: f, OnDisk: func(json.RawMessage) ([]byte, error) {
|
|
return nil, errors.New("permission denied")
|
|
}})
|
|
readErr.grants = &delivery.GrantsUserList{Follow: delivery.FollowGrants}
|
|
if _, err := readErr.Hold(cmp); err == nil || !strings.Contains(err.Error(), "could not be read") {
|
|
t.Fatalf("an unreadable list on the machine was held as %v", err)
|
|
}
|
|
}
|
|
}
|
|
if c.wantSource != "" && (st.Source != c.wantSource || st.Key != nodeKey.PublicKey ||
|
|
(c.wantFollows != "" && st.Follows != c.wantFollows)) {
|
|
t.Fatalf("the have says source %q follows %q key %t; want %q %q", st.Source, st.Follows, st.Key == nodeKey.PublicKey, c.wantSource, c.wantFollows)
|
|
}
|
|
switch {
|
|
case c.wantHanded == 0 && st.Handed != nil:
|
|
t.Fatalf("handed with no vault key or no hash: %+v", st.Handed)
|
|
case c.wantHanded > 0:
|
|
loaded := c.onDisk
|
|
if !c.wantRefused {
|
|
loaded = c.wantWrites[len(c.wantWrites)-1]
|
|
}
|
|
if st.Handed == nil || st.Handed.From != Digest([]byte(loaded)) {
|
|
t.Fatalf("handed %+v, want from the list on the machine", st.Handed)
|
|
}
|
|
opened, err := delivery.OpenHanded(*st.Handed, vault.PublicKey, vault.PrivateKey)
|
|
if err != nil || len(opened) != c.wantHanded || opened[0].User != "controller" {
|
|
t.Fatalf("opened %+v, %v", opened, err)
|
|
}
|
|
}
|
|
if c.wantRefused {
|
|
if !strings.Contains(st.Error, c.wantErr) || st.Digest != Digest([]byte(c.onDisk)) || st.Users != c.wantUsers {
|
|
t.Fatalf("refused list's state %+v", st)
|
|
}
|
|
return
|
|
}
|
|
if len(c.wantWrites) == 0 {
|
|
if module != "" || len(reports) != 0 {
|
|
t.Fatalf("a node with no list wrote or said something: %q %+v", module, reports)
|
|
}
|
|
return
|
|
}
|
|
if module != "nats" || st.Users != c.wantUsers || st.Wanted != Digest([]byte(c.wantWrites[len(c.wantWrites)-1])) {
|
|
t.Errorf("state %q %+v", module, st)
|
|
}
|
|
if c.wantErr != "" {
|
|
if !strings.Contains(st.Error, c.wantErr) || st.Digest != "" {
|
|
t.Errorf("a failed write is not kept: %+v", st)
|
|
}
|
|
} else if st.Digest != st.Wanted || st.Error != "" {
|
|
t.Errorf("written %+v", st)
|
|
}
|
|
if len(reports) == 0 || reports[len(reports)-1] != st {
|
|
t.Errorf("reports %+v, last state %+v", reports, st)
|
|
}
|
|
})
|
|
}
|
|
}
|