329 lines
19 KiB
Go
329 lines
19 KiB
Go
package placement
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"reflect"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
client "git.novox.be/novox/mesh-contracts/placement/go/clients/meshmoduleassignerclient"
|
|
seat "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner"
|
|
"git.novox.be/novox/mesh-sdk/go/keys"
|
|
)
|
|
|
|
// ADR 0567 §2, §4: what a change says, the holder's check of every placement, a proposal's changes laid over the keys,
|
|
// its digests (held to the values the earlier holder's code made, in the field order it wrote), its refusals, and who holds
|
|
// which seat.
|
|
func TestPlacement(t *testing.T) {
|
|
g14 := seat.Placement{Node: "g14", Revision: "abc",
|
|
Decided: seat.NodeDecided{Endpoint: "g14.example:51820", Site: "home", Address: "10.77.0.2", Account: "op", Taken: []string{"zsh"},
|
|
Tunnel: &seat.FoundTunnel{Interface: "wg0", Config: "c", Port: 51820, Mtu: 1420, PublicKey: "K", Peers: []seat.FoundTunnelPeer{{PublicKey: "P", Address: "10.0.0.3"}}},
|
|
PeerNames: map[string]string{"P": "pi"}},
|
|
Modules: []seat.PlacedModule{
|
|
{Module: "zsh"},
|
|
{Module: "app", Ports: map[string]int64{"8080": 18080}, Bindings: map[string]seat.Provider{"postgres": {Node: "novox", Module: "postgres"}}},
|
|
},
|
|
Pins: map[string]seat.Provider{"shell": {Node: "g14", Module: "zsh"}},
|
|
Holds: []seat.Holding{{Claim: "node-shell", Scope: "node", Module: "zsh"}},
|
|
}
|
|
novox := seat.Placement{Node: "novox", Decided: seat.NodeDecided{Hub: true, Address: "10.77.0.1"}, Modules: []seat.PlacedModule{{Module: "postgres", Ports: map[string]int64{"5432": 5432}}},
|
|
Holds: []seat.Holding{{Claim: "mesh-store", Scope: "mesh", Module: "postgres"}}}
|
|
current := map[string]seat.Placement{"g14": g14, "novox": novox}
|
|
const id = "p-20261028t183000-1a2b3c"
|
|
// the enroller is whichever module holds the seat mesh-node-enroller: no module is named for it
|
|
const enroller = "enroll-lab"
|
|
enrollers := EnrollersOf(map[string]seat.Placement{"ace": {Node: "ace",
|
|
Holds: []seat.Holding{{Claim: "mesh-node-enroller", Scope: "mesh", Module: enroller}}}})
|
|
if !slices.Equal(enrollers, []string{enroller}) {
|
|
t.Fatalf("the holders of the enroller seat = %v", enrollers)
|
|
}
|
|
|
|
t.Run("what a change says", func(t *testing.T) {
|
|
after := g14
|
|
after.Modules = []seat.PlacedModule{{Module: "app", Ports: map[string]int64{"8080": 18081}}, {Module: "new"}}
|
|
got := ChangeOf(g14, after)
|
|
want := Change{Node: "g14", Added: []string{"new"}, Removed: []string{"zsh"}, Changed: []string{"app"}}
|
|
if !reflect.DeepEqual(got, want) {
|
|
t.Errorf("ChangeOf = %+v, want %+v", got, want)
|
|
}
|
|
pinned := g14
|
|
pinned.Pins = nil
|
|
if got := ChangeOf(g14, pinned); !reflect.DeepEqual(got.Changed, []string{"app", "zsh"}) {
|
|
t.Errorf("a pin change changes every module on the node: %+v", got)
|
|
}
|
|
if got := ChangeOf(seat.Placement{}, novox); !reflect.DeepEqual(got.Added, []string{"postgres"}) || got.Node != "novox" {
|
|
t.Errorf("a new node: %+v", got)
|
|
}
|
|
c := []Change{{Node: "novox"}, {Node: "g14", Before: 1, After: 2}}
|
|
SortChanges(c)
|
|
if c[0].Node != "g14" || c[0].Body()["after"] != float64(2) {
|
|
t.Errorf("sorted %+v, body %v", c, c[0].Body())
|
|
}
|
|
})
|
|
|
|
t.Run("the check of every placement", func(t *testing.T) {
|
|
known := func(m string) bool { return m != "ghost" }
|
|
enrolled := func(n string) bool { return n != "stranger" }
|
|
if r := CheckPlacements(current, known, enrolled); len(r) != 0 {
|
|
t.Fatalf("a sound placement is refused: %v", r)
|
|
}
|
|
// A replicated mesh seat (ADR 0223) is held on two nodes; a seat that is not replicated is refused there.
|
|
for name, refused := range map[string]bool{"mesh-dns-resolver": false, "mesh-broker": true} {
|
|
a := seat.Placement{Node: "ace", Modules: []seat.PlacedModule{{Module: "dns"}}, Holds: []seat.Holding{{Claim: name, Scope: "mesh", Module: "dns"}}}
|
|
b := a
|
|
b.Node = "novox"
|
|
r := CheckPlacements(map[string]seat.Placement{"ace": a, "novox": b}, known, enrolled)
|
|
if (len(r) != 0) != refused {
|
|
t.Errorf("%s on two nodes: refusals %v, want refused %v", name, r, refused)
|
|
}
|
|
}
|
|
clash := g14
|
|
clash.Modules = append([]seat.PlacedModule{{Module: "ghost", Ports: map[string]int64{"80": 18080}}}, g14.Modules...)
|
|
clash.Modules[2] = seat.PlacedModule{Module: "app", Ports: map[string]int64{"8080": 18080}, Bindings: map[string]seat.Provider{"postgres": {Node: "ace", Module: "postgres"}}}
|
|
clash.Pins = map[string]seat.Provider{"shell": {Node: "g14", Module: "bash"}}
|
|
// ADR 0564 clause 6, Limits 5 (issue 1952 S8): mesh-watcher and mesh-self-check are never placed on one node.
|
|
clash.Modules = append(clash.Modules, seat.PlacedModule{Module: "mesh-watcher"}, seat.PlacedModule{Module: "mesh-self-check"})
|
|
twice := novox
|
|
twice.Node = "stranger"
|
|
twice.Decided.Address = "10.77.0.2"
|
|
r := CheckPlacements(map[string]seat.Placement{"g14": clash, "novox": novox, "stranger": twice}, known, enrolled)
|
|
for _, want := range []string{"holds no module ghost", "ghost and app are both published on host port 18080",
|
|
"bound to ace/postgres", "pin of shell names g14/bash", "stranger is a node with no record",
|
|
"mesh-store, held once for the mesh, is held on novox and stranger",
|
|
"g14 and stranger both have the address 10.77.0.2", "novox, stranger are each placed as it",
|
|
"g14: mesh-watcher and mesh-self-check are never placed on one node (ADR 0227 §6, ADR 0564 clause 6)"} {
|
|
if !slices.ContainsFunc(r, func(line string) bool { return strings.Contains(line, want) }) {
|
|
t.Errorf("no refusal says %q; got %v", want, r)
|
|
}
|
|
}
|
|
})
|
|
|
|
t.Run("changes laid over the keys", func(t *testing.T) {
|
|
laid, findings := LayChanges(current, []seat.ProposalChange{
|
|
{Op: client.OpPlace, Node: "g14", Module: "new"},
|
|
{Op: client.OpTakeOff, Node: "g14", Module: "zsh"},
|
|
{Op: client.OpUnpin, Node: "g14", Provision: "shell"},
|
|
{Op: client.OpDecide, Node: "novox", Field: "site", Value: "attic"},
|
|
})
|
|
if len(findings) != 0 || len(laid) != 2 || !reflect.DeepEqual(client.Assigned(laid["g14"]), []string{"app", "new"}) ||
|
|
len(laid["g14"].Holds) != 0 || laid["g14"].Pins != nil || laid["novox"].Decided.Site != "attic" || !laid["novox"].Decided.Hub {
|
|
t.Fatalf("laid %+v, %v", laid, findings)
|
|
}
|
|
if !reflect.DeepEqual(client.Assigned(current["g14"]), []string{"zsh", "app"}) {
|
|
t.Error("laying changed the current placement")
|
|
}
|
|
if fresh, f := LayChanges(current, []seat.ProposalChange{{Op: client.OpPlace, Node: "pi", Module: "zsh"}}); len(f) != 0 || fresh["pi"].Node != "pi" {
|
|
t.Errorf("a node with no key: %+v, %v", fresh, f)
|
|
}
|
|
for name, c := range map[string]struct {
|
|
change seat.ProposalChange
|
|
want string
|
|
}{
|
|
"not placed": {seat.ProposalChange{Op: client.OpTakeOff, Node: "g14", Module: "nope"}, "not placed"},
|
|
"no such field": {seat.ProposalChange{Op: client.OpDecide, Node: "g14", Field: "colour", Value: "x"}, "no decided field"},
|
|
"wrong type": {seat.ProposalChange{Op: client.OpDecide, Node: "g14", Field: "hub", Value: "x"}, "does not take"},
|
|
"unknown op": {seat.ProposalChange{Op: "move", Node: "g14"}, "no change of placement"},
|
|
"pin of nobody": {seat.ProposalChange{Op: client.OpPin, Node: "g14", Provision: "x"}, "provider"},
|
|
"not held": {seat.ProposalChange{Op: client.OpUnhold, Node: "g14", Module: "app", Claim: "x"}, "holds no"},
|
|
"bad port": {seat.ProposalChange{Op: client.OpPlace, Node: "g14", Module: "x", Ports: map[string]int64{"a": 1}}, "port"},
|
|
"seed of another": {seat.ProposalChange{Op: client.OpSeed, Node: "g14", Whole: &novox}, "seed"},
|
|
} {
|
|
if _, f := LayChanges(current, []seat.ProposalChange{c.change}); len(f) != 1 || !strings.Contains(f[0], c.want) {
|
|
t.Errorf("%s: %v, want a finding naming %q", name, f, c.want)
|
|
}
|
|
}
|
|
})
|
|
|
|
t.Run("a proposal, its digests and its refusals", func(t *testing.T) {
|
|
if got := NewProposalID(time.Date(2026, 10, 28, 18, 30, 0, 0, time.UTC)); !strings.HasPrefix(got, "p-20261028t183000-") || client.CheckProposalID(got) != nil {
|
|
t.Errorf("NewProposalID = %q", got)
|
|
}
|
|
p := seat.Proposal{ID: id, By: "operator@anchor", Why: "move the app", At: time.Unix(1, 0).UTC(),
|
|
Changes: []seat.ProposalChange{{Op: client.OpTakeOff, Node: "g14", Module: "zsh"}}, Base: map[string]int64{"g14": 4},
|
|
State: client.ProposalAnswering}
|
|
raw, err := EncodeProposal(p)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if back, err := DecodeProposal(raw); err != nil || back.ID != id || back.Base["g14"] != 4 || !reflect.DeepEqual(Nodes(back), []string{"g14"}) {
|
|
t.Fatalf("round trip %+v, %v", back, err)
|
|
}
|
|
if _, err := DecodeProposal([]byte(`{"v":9}`)); !errors.Is(err, keys.ErrUnknownVersion) {
|
|
t.Errorf("version 9: %v", err)
|
|
}
|
|
seed := p
|
|
seed.Seed, seed.ExportDigest = true, ExportDigest(current)
|
|
seed.Changes = []seat.ProposalChange{{Op: client.OpSeed, Node: "g14", Whole: &g14}}
|
|
if _, err := EncodeProposal(seed); err != nil {
|
|
t.Errorf("the seed: %v", err)
|
|
}
|
|
back := p
|
|
back.HandBack, back.Changes = true, nil
|
|
if _, err := EncodeProposal(back); err != nil {
|
|
t.Errorf("a hand-back: %v", err)
|
|
}
|
|
// The digests are what the earlier holder's code made of the same values (issue 1966: a proposal's ask binds its
|
|
// changes' digest, and a seed its export's), whatever order the generated types keep their fields in.
|
|
rich := seat.Proposal{ID: id, By: "operator@anchor", Why: "w", At: time.Unix(1, 0).UTC(), State: client.ProposalAnswering,
|
|
Base: map[string]int64{"g14": 4, "lab": 0, "novox": 2},
|
|
Changes: []seat.ProposalChange{
|
|
{Op: client.OpTakeOff, Node: "g14", Module: "zsh"},
|
|
{Op: client.OpPlace, Node: "g14", Module: "new", Ports: map[string]int64{"80": 8080}, Bindings: map[string]seat.Provider{"pg": {Node: "novox", Module: "postgres"}}},
|
|
{Op: client.OpPin, Node: "g14", Provision: "shell", Provider: &seat.Provider{Node: "g14", Module: "bash"}},
|
|
{Op: client.OpHold, Node: "g14", Module: "app", Claim: "node-shell", Scope: "node", Kind: "k", Site: "s"},
|
|
{Op: client.OpDecide, Node: "novox", Field: "site", Value: "attic"},
|
|
{Op: client.OpDecide, Node: "novox", Field: "peer-names", Value: map[string]any{"k": "v"}},
|
|
{Op: client.OpConfig, Node: "g14", Module: "app", MeshWide: map[string]any{"a": float64(1)}, Layer: map[string]any{"b": "c"}, MeshWideValues: map[string]any{"d": true}, Values: map[string]any{"e": []any{"f"}}},
|
|
{Op: client.OpAddNode, Node: "lab", Decided: &seat.NodeDecided{Address: "10.0.0.7", Account: "x", AgentAccountHome: "/h"}, Keys: &seat.ProposalChangeKeys{Overlay: "wg", Identity: "id", Sealing: "se"}},
|
|
}}
|
|
for name, c := range map[string]struct{ got, want string }{
|
|
"export": {ExportDigest(current), "sha256:742f03d8790d7efa6abff76636f4a69ba2704d646233960f52ce08604b96e128"},
|
|
"rich changes": {ChangesDigest(rich), "sha256:056338a2143d8c1af4e1275f41ffd06f18f3af3396a196d4444df83a0434475c"},
|
|
"take-off": {ChangesDigest(p), "sha256:7ce48744dea7934e6f3a6a1eeffb28df4096ee6a6a6b4ae29e15fb019044d8de"},
|
|
"seed": {ChangesDigest(seed), "sha256:731fe40d188acdf4a27ea73f3a5f1f9d584c3a8af7eaf15d4fb18e5da4853515"},
|
|
"no changes": {ChangesDigest(seat.Proposal{ID: id, Base: map[string]int64{"g14": 4}}), "sha256:3c5f1509c2d1c867954c47303e8b8d84bd4cd9aa3458464c21933c6496999bee"},
|
|
} {
|
|
if c.got != c.want {
|
|
t.Errorf("the digest of %s is %s, and the earlier holder made %s", name, c.got, c.want)
|
|
}
|
|
}
|
|
if ExportDigest(current) != ExportDigest(map[string]seat.Placement{"novox": novox, "g14": g14}) || ExportDigest(current) == ExportDigest(nil) {
|
|
t.Error("the export digest depends on order or not on the placements")
|
|
}
|
|
// 774/S42/AC1 (ADR 0587 §7): an add-node carries its enrolment's warrant naming its node and address, as its one
|
|
// change, from the enroller alone; a warrant naming another node or address is refused.
|
|
// At the operator's yes it carries the tunnel key alone (the operator, 2026-11-02 01:05).
|
|
joining := seat.ProposalChange{Op: client.OpAddNode, Node: "lab", Decided: &seat.NodeDecided{Address: "10.0.0.7"},
|
|
Keys: &seat.ProposalChangeKeys{Overlay: "wg"}}
|
|
add := seat.Proposal{ID: id, By: "ace.mesh-enrolment", Why: "lab joins", At: time.Unix(1, 0).UTC(),
|
|
Changes: []seat.ProposalChange{joining}, Base: map[string]int64{"lab": 0}, State: client.ProposalAnswering,
|
|
Warrant: &seat.Warrant{Node: "lab", Address: "10.0.0.7", Ask: "a-1", Warrant: "w-1"}}
|
|
if raw, err := EncodeProposal(add); err != nil {
|
|
t.Errorf("an add-node: %v", err)
|
|
} else if back, err := DecodeProposal(raw); err != nil || back.Warrant == nil || back.Changes[0].Keys.Overlay != "wg" {
|
|
t.Errorf("an add-node round trip: %+v, %v", back, err)
|
|
}
|
|
if AddNodeRefusal(add, enroller, enrollers) != "" || !strings.Contains(AddNodeRefusal(add, "mesh-delivery-assigner", enrollers), "enroll-lab's alone") {
|
|
t.Errorf("add-node by caller: %q, %q", AddNodeRefusal(add, enroller, enrollers), AddNodeRefusal(add, "mesh-delivery-assigner", enrollers))
|
|
}
|
|
if laid, f := LayChanges(current, add.Changes); len(f) != 0 || laid["lab"].Decided.Address != "10.0.0.7" || len(laid["lab"].Modules) != 0 {
|
|
t.Errorf("an add-node laid: %+v, %v", laid, f)
|
|
}
|
|
if _, f := LayChanges(current, []seat.ProposalChange{{Op: client.OpAddNode, Node: "g14", Decided: &seat.NodeDecided{Address: "10.0.0.9"},
|
|
Keys: &seat.ProposalChangeKeys{Overlay: "wg"}}}); len(f) != 1 || !strings.Contains(f[0], "placed already") {
|
|
t.Errorf("an add-node of a placed node: %v", f)
|
|
}
|
|
for name, mut := range map[string]func(*seat.Proposal){
|
|
"warrant of another node": func(p *seat.Proposal) {
|
|
p.Warrant = &seat.Warrant{Node: "pi", Address: "10.0.0.7", Ask: "a", Warrant: "w"}
|
|
},
|
|
"warrant of another address": func(p *seat.Proposal) {
|
|
p.Warrant = &seat.Warrant{Node: "lab", Address: "10.0.0.8", Ask: "a", Warrant: "w"}
|
|
},
|
|
"no warrant": func(p *seat.Proposal) { p.Warrant = nil },
|
|
"no tunnel key": func(p *seat.Proposal) {
|
|
c := joining
|
|
c.Keys = &seat.ProposalChangeKeys{Identity: "id", Sealing: "seal"}
|
|
p.Changes = []seat.ProposalChange{c}
|
|
},
|
|
"half its identity keys": func(p *seat.Proposal) {
|
|
c := joining
|
|
c.Keys = &seat.ProposalChangeKeys{Overlay: "wg", Identity: "id"}
|
|
p.Changes = []seat.ProposalChange{c}
|
|
},
|
|
"beside another change": func(p *seat.Proposal) {
|
|
p.Changes = append(p.Changes, seat.ProposalChange{Op: client.OpPlace, Node: "lab", Module: "zsh"})
|
|
},
|
|
"the hub": func(p *seat.Proposal) {
|
|
c := joining
|
|
c.Decided = &seat.NodeDecided{Address: "10.0.0.7", Hub: true}
|
|
p.Changes = []seat.ProposalChange{c}
|
|
},
|
|
"a warrant on a move": func(p *seat.Proposal) {
|
|
p.Changes, p.Base = []seat.ProposalChange{{Op: client.OpTakeOff, Node: "g14", Module: "zsh"}}, map[string]int64{"g14": 4}
|
|
},
|
|
} {
|
|
q := add
|
|
q.Changes = append([]seat.ProposalChange(nil), add.Changes...)
|
|
mut(&q)
|
|
if _, err := EncodeProposal(q); err == nil {
|
|
t.Errorf("an add-node with %s: taken", name)
|
|
}
|
|
}
|
|
// With its identity keys whole, as bound at redeem, it is taken too.
|
|
whole := add
|
|
whole.Changes = []seat.ProposalChange{joining}
|
|
whole.Changes[0].Keys = &seat.ProposalChangeKeys{Overlay: "wg", Identity: "id", Sealing: "seal"}
|
|
if _, err := EncodeProposal(whole); err != nil {
|
|
t.Errorf("an add-node with its identity keys: %v", err)
|
|
}
|
|
// An enrolment's end takes its never-confirmed node off again: the enroller's alone, of a node running nothing.
|
|
remove := add
|
|
remove.Changes = []seat.ProposalChange{{Op: client.OpRemoveNode, Node: "lab"}}
|
|
if _, err := EncodeProposal(remove); err != nil || AddNodeRefusal(remove, "mesh-delivery-assigner", enrollers) == "" ||
|
|
AddNodeRefusal(remove, enroller, enrollers) != "" || !slices.Equal(RemovedNodes(remove), []string{"lab"}) {
|
|
t.Errorf("a remove-node: %v, %q", err, AddNodeRefusal(remove, "mesh-delivery-assigner", enrollers))
|
|
}
|
|
joined := map[string]seat.Placement{"lab": {V: keys.Version, Node: "lab", Decided: seat.NodeDecided{Address: "10.0.0.7"}, Modules: []seat.PlacedModule{}}}
|
|
if laid, f := LayChanges(joined, remove.Changes); len(f) != 0 || len(laid) != 0 {
|
|
t.Errorf("a joining node removed: %v, %v", laid, f)
|
|
}
|
|
if _, f := LayChanges(current, []seat.ProposalChange{{Op: client.OpRemoveNode, Node: "g14"}}); len(f) != 1 ||
|
|
!strings.Contains(f[0], "runs modules") {
|
|
t.Errorf("a running node removed: %v", f)
|
|
}
|
|
remove.Warrant = &seat.Warrant{Node: "pi", Address: "10.0.0.7", Ask: "a", Warrant: "w"}
|
|
if _, err := EncodeProposal(remove); err == nil {
|
|
t.Error("a remove-node on another node's warrant: taken")
|
|
}
|
|
for name, mut := range map[string]func(*seat.Proposal){
|
|
"no id": func(p *seat.Proposal) { p.ID = "" },
|
|
"no proposer": func(p *seat.Proposal) { p.By = "" },
|
|
"no why": func(p *seat.Proposal) { p.Why = " " },
|
|
"no change": func(p *seat.Proposal) { p.Changes = nil },
|
|
"no base": func(p *seat.Proposal) { p.Base = map[string]int64{"novox": 1} },
|
|
"no state": func(p *seat.Proposal) { p.State = "" },
|
|
"asked, no ask": func(p *seat.Proposal) { p.State = client.ProposalAsked },
|
|
"seed mixed": func(p *seat.Proposal) { p.Seed, p.ExportDigest = true, "d" },
|
|
"bad change": func(p *seat.Proposal) { p.Changes[0].Op = "swap" },
|
|
"seed, no digest": func(p *seat.Proposal) { *p = seed; p.ExportDigest = "" },
|
|
"hand-back with changes": func(p *seat.Proposal) { p.HandBack = true },
|
|
} {
|
|
q := p
|
|
q.Changes = append([]seat.ProposalChange(nil), p.Changes...)
|
|
mut(&q)
|
|
if _, err := EncodeProposal(q); err == nil {
|
|
t.Errorf("%s: taken", name)
|
|
}
|
|
}
|
|
})
|
|
|
|
// ADR 0571 clause 4: who holds which seat, from placement; a node seat by its node.
|
|
t.Run("who holds which seat", func(t *testing.T) {
|
|
held := SeatHoldersOf(map[string]seat.Placement{
|
|
"ace": {Node: "ace", Holds: []seat.Holding{{Claim: "mesh-dns-resolver", Scope: "mesh", Module: "dnsmasq"}, {Claim: "node-files", Scope: "node", Module: "mesh-files"}}},
|
|
"novox": {Node: "novox", Holds: []seat.Holding{{Claim: "mesh-dns-resolver", Scope: "mesh", Module: "dnsmasq"}, {Claim: "mesh-broker", Scope: "mesh", Module: "nats"},
|
|
{Claim: "photo-gallery", Scope: "mesh", Module: "photos"}}},
|
|
})
|
|
if o := held["photo-gallery"]; !o.Outside || len(o.Holders) != 1 || held["mesh-broker"].Outside {
|
|
t.Errorf("a claim outside the set is %+v", o)
|
|
}
|
|
keyed := map[string][]byte{}
|
|
for k, v := range held {
|
|
keyed[k], _ = json.Marshal(v)
|
|
}
|
|
if h, err := client.HoldersOf(keyed, "mesh-dns-resolver", ""); err != nil || len(h) != 2 || h[0].Node != "ace" {
|
|
t.Errorf("a replicated seat's holders are %v, %v", h, err)
|
|
}
|
|
if h, _ := client.HoldersOf(keyed, "node-files", "ace"); len(h) != 1 || h[0].Module != "mesh-files" || len(held) != 4 {
|
|
t.Errorf("a node seat's holder is %v, keys %v", h, held)
|
|
}
|
|
if h, _ := client.HoldersOf(keyed, "node-files", "novox"); h != nil {
|
|
t.Errorf("a node seat nobody holds there is held by %v", h)
|
|
}
|
|
})
|
|
}
|