jschoubben
06f70e11ee
Merges fix/1945-adopt-hashes 1a190090. A new follower (one per link) waits for the grants part's key to catch up before
its first write, so a restart after take-over never writes the controller's list first. A refused or ended watch is
tried again on link.Backoff, the have says why it is not watched, and the key last heard, kept beside the node's state,
holds meanwhile. While the grants part's list is followed, the bus module's composition is applied with the list on the
machine in place of the controller's. The grants list is never handed to the vault. Pins mesh-sdk c2a410bb.
501 lines
20 KiB
Go
501 lines
20 KiB
Go
package userlist
|
|
|
|
import (
|
|
"context"
|
|
"crypto/ed25519"
|
|
"encoding/json"
|
|
"errors"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
"git.novox.be/novox/mesh-sdk/go/seal"
|
|
"github.com/novox/mesh-host/internal/agent"
|
|
)
|
|
|
|
const node = "novox"
|
|
|
|
func key() (ed25519.PublicKey, ed25519.PrivateKey) {
|
|
pub, k, err := ed25519.GenerateKey(nil)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
return pub, k
|
|
}
|
|
|
|
var controllerPub, controllerKey = key()
|
|
var _, strangerKey = key()
|
|
|
|
func fpOf(c string) string { return delivery.SourcePrefix2 + strings.Repeat(c, 64) }
|
|
|
|
// feed is a watch the test writes to: the keys there before the watch, Caught, then each put. watching is closed once
|
|
// the watch began, so a put after it is never handed before the keys there before it.
|
|
type feed struct {
|
|
ch chan agent.Update
|
|
rev uint64
|
|
first []agent.Update
|
|
// watching is closed once the watch caught up, so a later put is not read before the keys there before it.
|
|
watching chan struct{}
|
|
once sync.Once
|
|
// err refuses the watch, as the bus refuses a watch it grants nothing of.
|
|
err error
|
|
}
|
|
|
|
func (f *feed) Watch(context.Context, string) (<-chan agent.Update, error) {
|
|
if f.err != nil {
|
|
return nil, f.err
|
|
}
|
|
for _, u := range f.first {
|
|
f.ch <- u
|
|
}
|
|
f.ch <- agent.Update{Caught: true}
|
|
f.once.Do(func() {
|
|
if f.watching != nil {
|
|
close(f.watching)
|
|
}
|
|
})
|
|
return f.ch, nil
|
|
}
|
|
|
|
// hashed is a list whose users carry a hash of bcrypt's form, as the composer writes them.
|
|
func hashed(users ...string) string {
|
|
var b strings.Builder
|
|
b.WriteString("authorization {\n users = [\n")
|
|
for _, u := range users {
|
|
b.WriteString(` { user: "` + u + `", password: "$2a$11$` + strings.Repeat("h", 53) + `", permissions: {` + "\n")
|
|
}
|
|
b.WriteString(" ]\n}\n")
|
|
return b.String()
|
|
}
|
|
|
|
var vault, _ = seal.GenerateKeyPair()
|
|
|
|
func list(users ...string) string {
|
|
var b strings.Builder
|
|
b.WriteString("authorization {\n users = [\n")
|
|
for _, u := range users {
|
|
b.WriteString(` {user: "` + u + `", password: "$2a$11$x"}` + "\n")
|
|
}
|
|
b.WriteString(" ]\n}\n")
|
|
return b.String()
|
|
}
|
|
|
|
// comp is a composition of module at fp holding the user list content (none when empty), signed by signer; by digest
|
|
// when byDigest, as the controller sends a file over 64 KiB; for another node or module than its key when asked.
|
|
func comp(t *testing.T, module, fp, content string, signer ed25519.PrivateKey, p put) (string, []byte) {
|
|
t.Helper()
|
|
owner := module
|
|
if p.asModule != "" {
|
|
owner = p.asModule
|
|
}
|
|
server, _ := json.Marshal(map[string]any{"id": owner + ".server-conf", "type": "file", "module": owner,
|
|
"path": "/var/lib/nats-module/conf/nats.conf", "content": "include accounts.conf\n"})
|
|
rs := []json.RawMessage{server}
|
|
if content != "" {
|
|
r := map[string]any{"id": owner + ".bus-users", "type": "file", "module": owner,
|
|
"path": "/var/lib/nats-module/conf/accounts.conf", "content": content, "mode": "0600"}
|
|
if p.byDigest {
|
|
delete(r, "content")
|
|
r["digest"], r["size"] = Digest([]byte(content)), len(content)
|
|
}
|
|
users, _ := json.Marshal(r)
|
|
rs = append(rs, users)
|
|
}
|
|
c := delivery.Composed{Node: node, Module: module, FP: fp, Resources: rs}
|
|
if p.asNode != "" {
|
|
c.Node = p.asNode
|
|
}
|
|
if p.asModule != "" {
|
|
c.Module = p.asModule
|
|
}
|
|
k, err := delivery.ComposedKey(node, module, fp)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
signed, err := delivery.SignComposed(c, signer)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
raw, err := delivery.EncodeComposed(signed)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return k, raw
|
|
}
|
|
|
|
// gput is the grants part's key as the bus node hears it at the watch's start (ADR 0547 clauses 7, 8): which list to
|
|
// follow, and the list sealed to the node's key; one sealed to another key, or one that does not open to the key's digest.
|
|
type gput struct {
|
|
follow, content string
|
|
otherKey, badDigest bool
|
|
// after puts it once the grants part's list was written, as a hand-back comes after a take-over.
|
|
after bool
|
|
}
|
|
|
|
type put struct {
|
|
module, fp, content string
|
|
stranger, deleted bool
|
|
byDigest bool
|
|
asNode, asModule string
|
|
}
|
|
|
|
// Defends novox/hq ADR 0547 §8: the bus node writes the newest signed list alone, only when it changed, and reports it.
|
|
func TestTheBusNodeWritesTheNewestSignedUserListAlone(t *testing.T) {
|
|
one, two, three := list("controller", "a"), list("controller", "a", "b"), list("controller", "a", "b", "c")
|
|
five := list("controller", "a", "b", "c", "d")
|
|
for _, c := range []struct {
|
|
name string
|
|
before, after []put
|
|
again bool
|
|
failWrite bool
|
|
onDisk string
|
|
wantWrites []string // contents written, in order
|
|
wantUsers int
|
|
wantErr string
|
|
wantRefused bool
|
|
grants []gput
|
|
// grantsRefused refuses the watch of the grants part's key; kept is the key last heard, kept on the machine;
|
|
// wantUnwatched says the have then says why it is not watched.
|
|
grantsRefused bool
|
|
kept *gput
|
|
wantUnwatched bool
|
|
// wantSource and wantFollows are what the last have says it took the list from and follows.
|
|
wantSource, wantFollows string
|
|
// vault gives the follower the vault's key; wantHanded is how many users it then hands from the list on the
|
|
// machine (ADR 0547 clause 9).
|
|
vault bool
|
|
wantHanded int
|
|
}{
|
|
{name: "the list there is written once", before: []put{{module: "nats", fp: fpOf("a"), content: two}},
|
|
wantWrites: []string{two}, wantUsers: 3},
|
|
{name: "a newer revision is written, the older fingerprint is not",
|
|
before: []put{{module: "nats", fp: fpOf("a"), content: one}, {module: "nats", fp: fpOf("b"), content: two}},
|
|
after: []put{{module: "nats", fp: fpOf("a"), content: three}},
|
|
wantWrites: []string{two, three}, wantUsers: 4},
|
|
{name: "a composition signed by a stranger is no source",
|
|
before: []put{{module: "nats", fp: fpOf("a"), content: one}},
|
|
after: []put{{module: "nats", fp: fpOf("b"), content: three, stranger: true}},
|
|
wantWrites: []string{one}, wantUsers: 2},
|
|
{name: "a composition without the list is ignored", before: []put{{module: "web", fp: fpOf("a"), content: ""}}},
|
|
{name: "a deleted key is no source",
|
|
before: []put{{module: "nats", fp: fpOf("a"), content: one}, {module: "nats", fp: fpOf("b"), content: two}},
|
|
after: []put{{module: "nats", fp: fpOf("b"), content: "", deleted: true}, {module: "nats", fp: fpOf("a"), content: three}},
|
|
wantWrites: []string{two, three}, wantUsers: 4},
|
|
{name: "asked again, the same source is written again", before: []put{{module: "nats", fp: fpOf("a"), content: two}},
|
|
again: true, wantWrites: []string{two, two}, wantUsers: 3},
|
|
{name: "a list sent by digest counts the users of what it names",
|
|
before: []put{{module: "nats", fp: fpOf("a"), content: three, byDigest: true}},
|
|
wantWrites: []string{three}, wantUsers: 4},
|
|
{name: "another node's composition under this node's key is no source",
|
|
before: []put{{module: "nats", fp: fpOf("a"), content: two, asNode: "ace"}}},
|
|
{name: "another module's composition under the bus module's key is no source",
|
|
before: []put{{module: "nats", fp: fpOf("a"), content: two, asModule: "web"}}},
|
|
{name: "a list without the controller is refused and the file kept",
|
|
before: []put{{module: "nats", fp: fpOf("a"), content: list("a", "b")}},
|
|
onDisk: two, wantUsers: 2, wantErr: "holds no user \"controller\"", wantRefused: true},
|
|
{name: "a list that drops more than half is refused and the file kept",
|
|
before: []put{{module: "nats", fp: fpOf("a"), content: one}},
|
|
onDisk: five, wantUsers: 2, wantErr: "drops more than half", wantRefused: true},
|
|
{name: "a list that drops half is written", before: []put{{module: "nats", fp: fpOf("a"), content: two}},
|
|
onDisk: list("controller", "a", "b", "c", "d", "e"), wantWrites: []string{two}, wantUsers: 3},
|
|
{name: "the loaded list's hashes are handed sealed to the vault's key",
|
|
before: []put{{module: "nats", fp: fpOf("a"), content: hashed("controller", "node.ace")}}, vault: true,
|
|
wantWrites: []string{hashed("controller", "node.ace")}, wantUsers: 2, wantHanded: 2},
|
|
{name: "a refused list hands the list on the machine",
|
|
before: []put{{module: "nats", fp: fpOf("a"), content: list("a", "b")}}, vault: true,
|
|
onDisk: hashed("controller", "a", "b"), wantUsers: 2, wantErr: "holds no user", wantRefused: true, wantHanded: 3},
|
|
{name: "a failed write is said and kept", before: []put{{module: "nats", fp: fpOf("a"), content: two}}, failWrite: true,
|
|
wantWrites: []string{two}, wantUsers: 3, wantErr: "disk full"},
|
|
// ADR 0547 clauses 7, 8: never both lists; the grants part's from take-over, the controller's otherwise.
|
|
{name: "from take-over the grants part's list is written into the same file",
|
|
before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grants: []gput{{follow: delivery.FollowGrants, content: three}},
|
|
wantWrites: []string{three}, wantUsers: 4, wantSource: delivery.GrantsUserListSource, wantFollows: delivery.FollowGrants},
|
|
{name: "a hand-back writes the newest controller composition once",
|
|
before: []put{{module: "nats", fp: fpOf("a"), content: two}},
|
|
grants: []gput{{follow: delivery.FollowGrants, content: three}, {follow: delivery.FollowController, content: three, after: true}},
|
|
wantWrites: []string{three, two}, wantUsers: 3, wantSource: "novox.nats." + fpOf("a")[len(delivery.SourcePrefix2):], wantFollows: delivery.FollowController},
|
|
{name: "a refused watch with no key kept follows the controller's list and says it is not watched",
|
|
before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grantsRefused: true, wantUnwatched: true,
|
|
wantWrites: []string{two}, wantUsers: 3, wantSource: "novox.nats." + fpOf("a")[len(delivery.SourcePrefix2):], wantFollows: delivery.FollowController},
|
|
{name: "a refused watch keeps the follow last heard, never the controller's list",
|
|
before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grantsRefused: true, wantUnwatched: true,
|
|
kept: &gput{follow: delivery.FollowGrants, content: three},
|
|
wantWrites: []string{three}, wantUsers: 4, wantSource: delivery.GrantsUserListSource, wantFollows: delivery.FollowGrants},
|
|
{name: "in shadow the grants part's list is not taken",
|
|
before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grants: []gput{{follow: delivery.FollowController, content: three}},
|
|
wantWrites: []string{two}, wantUsers: 3, wantSource: "novox.nats." + fpOf("a")[len(delivery.SourcePrefix2):], wantFollows: delivery.FollowController},
|
|
{name: "a grants list that does not open to its digest is refused and the file kept",
|
|
before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grants: []gput{{follow: delivery.FollowGrants, content: three, badDigest: true}},
|
|
onDisk: two, wantErr: "could not be had", wantRefused: true, wantSource: delivery.GrantsUserListSource},
|
|
{name: "a grants list sealed to another key is refused and the file kept",
|
|
before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grants: []gput{{follow: delivery.FollowGrants, content: three, otherKey: true}},
|
|
onDisk: two, wantErr: "sealed to another key", wantRefused: true, wantSource: delivery.GrantsUserListSource},
|
|
} {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
f := &feed{ch: make(chan agent.Update, 64), watching: make(chan struct{})}
|
|
objects := map[string]string{}
|
|
var mu sync.Mutex
|
|
var writes []string
|
|
var reports []Written
|
|
reported := make(chan struct{}, 64)
|
|
gf := &feed{ch: make(chan agent.Update, 64)}
|
|
if c.grantsRefused {
|
|
gf.err = errors.New("Permissions Violation for Publish to $JS.API.STREAM.INFO.KV_mesh-grants_user-list")
|
|
}
|
|
var grantsAfter []agent.Update
|
|
var keptRaw []byte
|
|
sealed := map[string][]byte{}
|
|
nodeKey, err := seal.GenerateKeyPair()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
gs := c.grants
|
|
if c.kept != nil {
|
|
gs = append(append([]gput(nil), gs...), *c.kept)
|
|
}
|
|
for i, g := range gs {
|
|
to := nodeKey.PublicKey
|
|
if g.otherKey {
|
|
other, _ := seal.GenerateKeyPair()
|
|
to = other.PublicKey
|
|
}
|
|
digest := delivery.UserListDigest([]byte(g.content))
|
|
if g.badDigest {
|
|
digest = delivery.UserListDigest([]byte("another list"))
|
|
}
|
|
object, err := delivery.SealUserList([]byte(g.content), to)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sealed[delivery.UserListObjectName(digest)] = object
|
|
raw, err := delivery.EncodeGrantsUserList(delivery.GrantsUserList{Follow: g.follow, Digest: digest,
|
|
Object: delivery.UserListObjectName(digest), SealedTo: delivery.SealingKeyID(to), Node: node, Revision: uint64(i + 1)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
u := agent.Update{Key: delivery.GrantsUserListKey, Value: raw, Revision: uint64(i + 1)}
|
|
switch {
|
|
case c.kept != nil && i == len(gs)-1:
|
|
keptRaw = raw
|
|
case g.after:
|
|
grantsAfter = append(grantsAfter, u)
|
|
default:
|
|
gf.first = append(gf.first, u)
|
|
}
|
|
}
|
|
fl := New(Env{Node: node, Composed: f, Grants: gf,
|
|
Kept: func() ([]byte, uint64) { return keptRaw, 9 },
|
|
Keep: func([]byte, uint64) {},
|
|
Backoff: func(int) time.Duration { return time.Hour },
|
|
Fetch: func(_ context.Context, name string) ([]byte, error) {
|
|
if o, ok := sealed[name]; ok {
|
|
return o, nil
|
|
}
|
|
return nil, errors.New("no object " + name)
|
|
},
|
|
Open: func(b seal.Box) ([]byte, error) {
|
|
out, err := seal.Open(b, nodeKey.PrivateKey)
|
|
return []byte(out), err
|
|
},
|
|
Key: func() string { return nodeKey.PublicKey },
|
|
Trusted: func() []ed25519.PublicKey { return []ed25519.PublicKey{controllerPub} },
|
|
Resolve: func(_ context.Context, r json.RawMessage) ([]byte, error) {
|
|
var res struct{ Content, Digest string }
|
|
_ = json.Unmarshal(r, &res)
|
|
if res.Digest != "" {
|
|
body, ok := objects[res.Digest]
|
|
if !ok {
|
|
return nil, errors.New("no object " + res.Digest)
|
|
}
|
|
return []byte(body), nil
|
|
}
|
|
return []byte(res.Content), nil
|
|
},
|
|
OnDisk: func(json.RawMessage) ([]byte, error) {
|
|
if c.onDisk == "" {
|
|
return nil, nil
|
|
}
|
|
return []byte(c.onDisk), nil
|
|
},
|
|
Write: func(_ context.Context, module string, r json.RawMessage, body []byte) (string, bool, error) {
|
|
mu.Lock()
|
|
writes = append(writes, string(body))
|
|
mu.Unlock()
|
|
if c.failWrite {
|
|
return "", false, errors.New("disk full")
|
|
}
|
|
return Digest(body), true, nil
|
|
},
|
|
Report: func(_ context.Context, module string, w Written) {
|
|
mu.Lock()
|
|
reports = append(reports, w)
|
|
mu.Unlock()
|
|
reported <- struct{}{}
|
|
},
|
|
Now: func() time.Time { return time.Date(2026, 10, 25, 21, 50, 0, 0, time.UTC) },
|
|
VaultKey: func() string {
|
|
if c.vault {
|
|
return vault.PublicKey
|
|
}
|
|
return ""
|
|
},
|
|
})
|
|
update := func(p put) agent.Update {
|
|
f.rev++
|
|
signer := controllerKey
|
|
if p.stranger {
|
|
signer = strangerKey
|
|
}
|
|
k, raw := comp(t, p.module, p.fp, p.content, signer, p)
|
|
if p.byDigest {
|
|
objects[Digest([]byte(p.content))] = p.content
|
|
}
|
|
if p.deleted {
|
|
return agent.Update{Key: k, Revision: f.rev, Deleted: true}
|
|
}
|
|
return agent.Update{Key: k, Value: raw, Revision: f.rev}
|
|
}
|
|
for _, p := range c.before {
|
|
f.first = append(f.first, update(p))
|
|
}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
done := make(chan error, 1)
|
|
go func() { done <- fl.Run(ctx) }()
|
|
<-f.watching
|
|
// A later put after the first list is written: the first write waits for the grants part's key to catch up.
|
|
if len(c.after) > 0 {
|
|
for until := time.Now().Add(5 * time.Second); time.Now().Before(until); time.Sleep(5 * time.Millisecond) {
|
|
mu.Lock()
|
|
n := len(writes)
|
|
mu.Unlock()
|
|
if n > 0 {
|
|
break
|
|
}
|
|
}
|
|
}
|
|
for _, p := range c.after {
|
|
f.ch <- update(p)
|
|
}
|
|
if len(grantsAfter) > 0 {
|
|
for until := time.Now().Add(5 * time.Second); ; time.Sleep(5 * time.Millisecond) {
|
|
if _, st := fl.State(); st.Source == delivery.GrantsUserListSource || time.Now().After(until) {
|
|
break
|
|
}
|
|
}
|
|
for _, u := range grantsAfter {
|
|
gf.ch <- u
|
|
}
|
|
}
|
|
if c.again {
|
|
// After the first write is reported, so the look again is not folded into it.
|
|
select {
|
|
case <-reported:
|
|
case <-time.After(5 * time.Second):
|
|
}
|
|
fl.Again()
|
|
}
|
|
deadline := time.After(5 * time.Second)
|
|
for {
|
|
mu.Lock()
|
|
n := len(writes)
|
|
mu.Unlock()
|
|
if n >= len(c.wantWrites) {
|
|
break
|
|
}
|
|
select {
|
|
case <-deadline:
|
|
t.Fatalf("writes %d, want %d", n, len(c.wantWrites))
|
|
case <-time.After(5 * time.Millisecond):
|
|
}
|
|
}
|
|
if c.wantRefused {
|
|
select {
|
|
case <-reported:
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("a refused list was not reported")
|
|
}
|
|
}
|
|
if c.wantSource != "" {
|
|
for until := time.Now().Add(5 * time.Second); ; {
|
|
if _, st := fl.State(); st.Source == c.wantSource && (c.wantErr == "" || st.Error != "") || time.Now().After(until) {
|
|
break
|
|
}
|
|
time.Sleep(5 * time.Millisecond)
|
|
}
|
|
}
|
|
time.Sleep(50 * time.Millisecond) // nothing more is written
|
|
cancel()
|
|
<-done
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
if strings.Join(writes, "|") != strings.Join(c.wantWrites, "|") {
|
|
t.Fatalf("written %q, want %q", writes, c.wantWrites)
|
|
}
|
|
module, st := fl.State()
|
|
if c.wantUnwatched != (st.Unwatched != "") {
|
|
t.Fatalf("the have says unwatched %q", st.Unwatched)
|
|
}
|
|
// While the grants part's list is followed, an apply of the controller's composition writes not its list.
|
|
if _, raw := comp(t, "nats", fpOf("a"), two, controllerKey, put{}); c.wantFollows != "" {
|
|
cmp, err := delivery.DecodeComposed(raw)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
held := fl.Hold(cmp)
|
|
_, _, _, hasList := userList(held.Resources)
|
|
if want := c.wantFollows == delivery.FollowController; hasList != want || len(held.Resources) < 1 {
|
|
t.Fatalf("an apply while following %s holds the controller's list: %v", c.wantFollows, hasList)
|
|
}
|
|
}
|
|
if c.wantSource != "" && (st.Source != c.wantSource || st.Key != nodeKey.PublicKey ||
|
|
(c.wantFollows != "" && st.Follows != c.wantFollows)) {
|
|
t.Fatalf("the have says source %q follows %q key %t; want %q %q", st.Source, st.Follows, st.Key == nodeKey.PublicKey, c.wantSource, c.wantFollows)
|
|
}
|
|
switch {
|
|
case c.wantHanded == 0 && st.Handed != nil:
|
|
t.Fatalf("handed with no vault key or no hash: %+v", st.Handed)
|
|
case c.wantHanded > 0:
|
|
loaded := c.onDisk
|
|
if !c.wantRefused {
|
|
loaded = c.wantWrites[len(c.wantWrites)-1]
|
|
}
|
|
if st.Handed == nil || st.Handed.From != Digest([]byte(loaded)) {
|
|
t.Fatalf("handed %+v, want from the list on the machine", st.Handed)
|
|
}
|
|
opened, err := delivery.OpenHanded(*st.Handed, vault.PublicKey, vault.PrivateKey)
|
|
if err != nil || len(opened) != c.wantHanded || opened[0].User != "controller" {
|
|
t.Fatalf("opened %+v, %v", opened, err)
|
|
}
|
|
}
|
|
if c.wantRefused {
|
|
if !strings.Contains(st.Error, c.wantErr) || st.Digest != Digest([]byte(c.onDisk)) || st.Users != c.wantUsers {
|
|
t.Fatalf("refused list's state %+v", st)
|
|
}
|
|
return
|
|
}
|
|
if len(c.wantWrites) == 0 {
|
|
if module != "" || len(reports) != 0 {
|
|
t.Fatalf("a node with no list wrote or said something: %q %+v", module, reports)
|
|
}
|
|
return
|
|
}
|
|
if module != "nats" || st.Users != c.wantUsers || st.Wanted != Digest([]byte(c.wantWrites[len(c.wantWrites)-1])) {
|
|
t.Errorf("state %q %+v", module, st)
|
|
}
|
|
if c.wantErr != "" {
|
|
if !strings.Contains(st.Error, c.wantErr) || st.Digest != "" {
|
|
t.Errorf("a failed write is not kept: %+v", st)
|
|
}
|
|
} else if st.Digest != st.Wanted || st.Error != "" {
|
|
t.Errorf("written %+v", st)
|
|
}
|
|
if len(reports) == 0 || reports[len(reports)-1] != st {
|
|
t.Errorf("reports %+v, last state %+v", reports, st)
|
|
}
|
|
})
|
|
}
|
|
}
|