novox/mesh-host
18
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request 'The node-engine trusts the vault alone, loads the grants part's user list alone and applies only its own compositions (issue 1126)' (#196) from fix/1126-gates-host into main

Browse Source
This commit was merged in pull request #196.
This commit is contained in:
mesh-tools
2026-11-02 09:51:15 +00:00
parent 14a12cc7b6 38d666f223
commit 84737f4cb2
13 changed files with 221 additions and 1145 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mesh-host/authority_test.go
+14 -21
View File
@@ -12,7 +12,6 @@ import (
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/overlay"
@@ -41,21 +40,24 @@ func TestTheAppliedAuthorityIsKeptForTheRecord(t *testing.T) {
t.Errorf("the record's keys read the authority %q", got.Authority)
}
// L3 s6: before the vault's signers are read the node trusts the controller's key alone; once read, a message the
// vault signed is accepted and the record names both; once the lease is handed away the controller's key is refused
// and the record names the vault's alone. Each change tells the record's writer.
// Issue 1126, A6 (ADR 0550 §2; the controller ends by its unassignment): a node enrolled with the controller's key
// trusts it never, whatever the lease says; before the vault's signers are read it trusts nothing, once read the
// vault's key alone, and the record names the vault's alone. Each change tells the record's writer.
controllerPub, controller, _ := ed25519.GenerateKey(nil)
vaultPub, vault, _ := ed25519.GenerateKey(nil)
trust := newMeshTrust(controllerPub)
trust := newMeshTrust()
told := 0
trust.onChange(func() { told++ })
keys := recordKeys(state, trust.Trusts)
ctrlFP, vaultFP := delivery.SignerFingerprint(controllerPub), delivery.SignerFingerprint(vaultPub)
if got := keys().Trusts; !delivery.TrustsKey(got, delivery.SignerController, ctrlFP) || delivery.TrustsKey(got, delivery.SignerVault, vaultFP) {
t.Errorf("before the signers are read the record trusts %v", got)
}
enrol, _ := delivery.SignContext(delivery.KindEnrolment)
msg := []byte("a reply")
if got := keys().Trusts; len(got) != 0 {
t.Errorf("before the signers are read the record trusts %v", got)
}
if _, err := trust.Verify(delivery.KindEnrolment, msg, delivery.SignMessage(controller, enrol, msg)); err == nil {
t.Error("a controller-signed message is accepted")
}
if _, err := trust.Verify(delivery.KindEnrolment, msg, delivery.SignMessage(vault, enrol, msg)); err == nil {
t.Error("a vault-signed message is accepted before the vault's key is read")
}
@@ -68,22 +70,13 @@ func TestTheAppliedAuthorityIsKeptForTheRecord(t *testing.T) {
if s, err := trust.Verify(delivery.KindEnrolment, msg, delivery.SignMessage(vault, enrol, msg)); err != nil || s.Name != delivery.SignerVault {
t.Errorf("a vault-signed message on a node that read signers: %+v, %v", s, err)
}
if got := keys().Trusts; !delivery.TrustsKey(got, delivery.SignerController, ctrlFP) || !delivery.TrustsKey(got, delivery.SignerVault, vaultFP) ||
if _, err := trust.Verify(delivery.KindEnrolment, msg, delivery.SignMessage(controller, enrol, msg)); err == nil {
t.Error("a controller-signed message is accepted once the vault's key is read")
}
if got := keys().Trusts; delivery.TrustsKey(got, delivery.SignerController, ctrlFP) || !delivery.TrustsKey(got, delivery.SignerVault, vaultFP) ||
trust.VaultAuthority() != vaultCA || told != 2 {
t.Errorf("after the signers are read the record trusts %v (told %d)", got, told)
}
handed, _ := json.Marshal(buscompose.LeaseHolder{HandedTo: "mesh-vault", HandedBy: "person.operator"})
trust.heardLease(handed, true, nil)
if _, err := trust.Verify(delivery.KindEnrolment, msg, delivery.SignMessage(controller, enrol, msg)); err == nil || trust.Controller() != nil {
t.Error("after the lease is handed away the controller's key is still trusted")
}
if got := keys().Trusts; delivery.TrustsKey(got, delivery.SignerController, ctrlFP) || !delivery.TrustsKey(got, delivery.SignerVault, vaultFP) || told != 3 {
t.Errorf("after the lease is handed away the record trusts %v (told %d)", got, told)
}
trust.heardLease(handed, true, nil)
if told != 3 {
t.Error("a lease heard again unchanged wrote the record again")
}
// A signers' key naming another fingerprint is not trusted.
bad, _ := json.Marshal(delivery.SignerKey{Key: base64.StdEncoding.EncodeToString(controllerPub), Fingerprint: vaultFP})
trust.heardSigner(delivery.SignersKeyKey, bad, false, func(string) {})
cmd/mesh-host/found.go
+5 -14
View File
@@ -64,27 +64,18 @@ func foundDirectories(known store.State) []store.Applied {
return out
}
// handOverLine is the operator's line that hands a directory to the mesh, with this node's name when it is
// known (novox/hq ADR 0272): typed on the control-node, from the operator's own account. The directory's
// path is the resource's target beside the reason, never in it: the controller says a reason on the
// operator's channel, which withholds a path (ADR 0234 §6), and keeps the target as evidence.
//
// From the lease write, heard as the engine hears it (meshTrust), the line has no answerer: the way out is the node
// seat's hand-over with the directory's path, which asks the operator on their channel (novox/hq ADR 0563 §5).
// handOverLine is the way out of a directory used as found, with this node's name when it is known: the node seat's
// hand-over with the directory's path, which asks the operator on their channel (novox/hq ADR 0563 §5). The controller
// that answered `nox node hand-over` ended by its unassignment (issue 1126), so that line has no answerer. The path is
// the resource's target beside the reason, never in it: the operator's channel withholds a path (ADR 0234 §6).
func handOverLine() string {
node, _ := foundNode.Load().(string)
if node == "" {
node = "<node>"
}
if t := foundTrust.Load(); t != nil && t.HandedAway() {
return fmt.Sprintf("`%s/node-engine.hand-over` with the directory's path, then your yes on your channel", node)
}
return fmt.Sprintf("`nox node hand-over %s <directory>` on the control-node", node)
return fmt.Sprintf("`%s/node-engine.hand-over` with the directory's path, then your yes on your channel", node)
}
// foundTrust is what the engine trusts, whose lease decides the way out a directory used as found names.
var foundTrust atomic.Pointer[meshTrust]
// withFoundDirectories adds to a statement every directory used as found, as its module's resource of kind
// directory, unhealthy: the reason names its owner and mode as they are now, what was declared, and the line
// that hands it over. The path is the target, which the controller keeps as evidence.
cmd/mesh-host/found_test.go
+6 -20
View File
@@ -15,7 +15,7 @@ import (
// Every directory used as found is its module's resource of kind directory, unhealthy, naming its owner, on
// every statement (novox/hq issue 339): the controller raises that as the module's condition. The way out it
// names is the operator's line, with this node's name (novox/hq ADR 0272, issue 356).
// names is the node seat's hand-over, with this node's name (novox/hq ADR 0563 §5).
func TestAFoundDirectoryIsStatedUnhealthy(t *testing.T) {
dir := t.TempDir()
known := store.State{Resources: []store.Applied{
@@ -35,29 +35,15 @@ func TestAFoundDirectoryIsStatedUnhealthy(t *testing.T) {
t.Fatalf("stated as: %+v", r)
}
// The path is the target, kept as evidence, and never in the reason: the operator's channel withholds a path
// (novox/hq ADR 0234 §6), and the controller says the reason there.
if !strings.Contains(r.Reason, "`nox node hand-over laptop <directory>` on the control-node") ||
// (novox/hq ADR 0234 §6). The way out is the node seat's hand-over (ADR 0563 §5), never the controller's terminal line,
// which nothing answers since the controller ended (issue 1126).
if !strings.Contains(r.Reason, "`laptop/node-engine.hand-over`") || strings.Contains(r.Reason, "nox node hand-over") ||
strings.Contains(r.Reason, "mesh-host") || strings.Contains(r.Reason, dir) {
t.Fatalf("the way out is not the operator's line, or carries the path: %q", r.Reason)
t.Fatalf("the way out is not the node seat's hand-over, or carries the path: %q", r.Reason)
}
// ADR 0563 §5: from the lease write the remedy is the node seat's hand-over, answered on the operator's channel, and
// no longer the controller's terminal line; the lease is the one the engine hears.
handed := newMeshTrust(nil)
handed.handedAway = true
foundTrust.Store(handed)
if r := withFoundDirectories(&link.Health{}, foundDirectories(known)).Resources[0]; !strings.Contains(r.Reason,
"`laptop/node-engine.hand-over`") || strings.Contains(r.Reason, "nox node hand-over") || strings.Contains(r.Reason, dir) {
t.Fatalf("after the lease write the way out is %q", r.Reason)
}
foundTrust.Store(newMeshTrust(nil))
if r := withFoundDirectories(&link.Health{}, foundDirectories(known)).Resources[0]; !strings.Contains(r.Reason,
"`nox node hand-over laptop <directory>`") {
t.Fatalf("while the controller holds its lease the way out is %q", r.Reason)
}
foundTrust.Store((*meshTrust)(nil))
foundNode.Store("")
if r := withFoundDirectories(&link.Health{}, foundDirectories(known)).Resources[0]; !strings.Contains(r.Reason,
"`nox node hand-over <node> <directory>`") {
"`<node>/node-engine.hand-over`") {
t.Fatalf("before enrolment the line names a placeholder: %q", r.Reason)
}
}
cmd/mesh-host/main.go
+6 -7
View File
@@ -1113,17 +1113,16 @@ func runLink(ctx context.Context, opts options) error {
// the bus, answer what the controller asks of this machine, and say its facts: when a link opens, and when the
// controller asks.
proof := &proving{statePath: opts.state, version: runningVersion(), say: say}
// What this engine trusts a mesh signature from: the controller's key until its lease is handed away, and the vault's
// once read from its signers (novox/hq ADR 0550 §2).
trust := newMeshTrust(mine.Membership.Signer)
// And the way out a directory used as found names follows the lease it hears (novox/hq ADR 0563 §5).
foundTrust.Store(trust)
// What this engine trusts a mesh signature from: the vault's key alone, once read from its signers (novox/hq ADR 0550
// §2; issue 1126: the controller's key never, the controller ended by its unassignment).
trust := newMeshTrust()
queue := &link.Session{
Membership: membership,
// And the public halves and certificate its record says (novox/hq ADR 0555 §3), read at each facts report.
Keys: recordKeys(opts.state, trust.Trusts),
// And the controller's key, read per signed ask: none once its lease is handed away (novox/hq ADR 0563 §7).
ControllerKey: trust.Controller,
// And no controller's key: every ask the controller would sign is refused, never verified with the membership's
// signer (novox/hq ADR 0563 §7; issue 1126, the controller ended by its unassignment).
ControllerKey: noControllerKey,
// And the first have this build writes is what proves it to the launcher (ADR 0539 §9, ADR 0227 §8).
HaveWritten: proof.haveWritten,
// And the operator's hand-over of a directory used as found, carried by the controller from the
cmd/mesh-host/nodeagent.go
+28 -163
View File
@@ -380,10 +380,11 @@ func resourcesOf(cs []delivery.Composed) []declaration.Resource {
// controller's key the node was enrolled with (novox/hq ADR 0539 §3, §11); a node adopted later has it at its adoption.
const operatorKeysFile = "operator-keys"
// trustedKeys are the keys a composition may be signed by: the controller's, which the node was enrolled with, while the
// controller holds its lease (ADR 0550 §2: trusted no more once it is handed away), and the operator's, read from
// operatorKeysFile each time it is asked, so a key that arrives while the agent runs is trusted at the next composition.
// A file that cannot be read trusts the controller's alone, and says so.
// trustedKeys are the keys a composition in the node's composition bucket may be signed by: the one given (none for the
// agent; the enrolment's signer for the user-list follower, which reads only where the bus's user-list file is from
// such a composition, never its list) and the operator's, read from operatorKeysFile each time it is asked, so a key
// that arrives while the agent runs is trusted at the next composition. A file that cannot be read trusts the one given
// alone, and says so.
func trustedKeys(opts options, controllerNow func() ed25519.PublicKey) func() []ed25519.PublicKey {
return func() []ed25519.PublicKey {
keys := []ed25519.PublicKey{}
@@ -509,9 +510,11 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus
}
}
}
// The bus's user list, written alone from the newest composition of the bus module (ADR 0547 §8): on every
// node, and a node whose compositions carry no list writes nothing.
users := userListFollower(opts, session, composedBucket, trustedKeys(opts, trust.Controller), say, func() {
// The bus's user list, the grants part's alone (ADR 0547 §8, clause 11; issue 1126), written into the file the
// bus module's composition in the node's bucket declares: on every node, and a node whose compositions carry no
// list writes nothing. Those compositions were signed with the enrolment's signer and nobody writes them now; they
// are read for the file's place alone, never for a list, until the place is read elsewhere (milestone B17).
users := userListFollower(opts, session, composedBucket, trustedKeys(opts, func() ed25519.PublicKey { return signer }), say, func() {
// This engine wrote the list, so the bus reloads its accounts and answers no JetStream request for a few
// seconds: what failed in them is tried again once that is over, not at its next wait.
time.AfterFunc(reloadSettles, func() {
@@ -520,8 +523,6 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus
}
})
})
// A vault key heard (or heard again changed) looks at the list again, so its hashes are handed (ADR 0547 §9).
vaultKey.following(users.Again)
go func() {
if err := users.Run(linked); err != nil && linked.Err() == nil {
say("the bus's user list is not followed on this link: " + err.Error())
@@ -551,8 +552,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus
}
return out, nil
},
Trusted: trustedKeys(opts, trust.Controller),
Backup: backupNode(session),
Backup: backupNode(session),
// What the filling refused before this apply may be named now: the provisioner's client is told to fill
// again (ace, 2026-10-25 21:17).
Settled: func(module string) {
@@ -603,8 +603,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus
return res
},
}
// The node composes its own modules once the catalogue places it (novox/hq ADR 0547 §2, ADR 0549, issue 1126): its
// placement key is read at the agent's start; until then it applies the controller's compositions as before.
// The node composes its own modules (novox/hq ADR 0547 §2, ADR 0549, issue 1126).
run, stopRun := context.WithCancel(linked)
// What the node runs and how it opens a sealed value, for the composer to keep the boxes it runs; and the agent
// told when the node cannot compose, so each wanted module's have says it.
@@ -649,8 +648,8 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus
shadowHooks := hooks
shadowHooks.refused, shadowHooks.certify = nil, nil
shadowEnv := composerEnv(session, node, say, shadowHooks)
// And, after each answer, what runs of an own secret the answer says the vault holds another value of, handed to
// the vault while the node applies the controller's compositions (issue 1126, cause C4; ownresync).
// And, after each answer, what runs of an own secret the answer says the vault holds another value of: handed to
// the vault only while a node applied the controller's compositions (issue 1126, cause C4; ownresync), so nothing now.
composes := &composesNow{}
resync := ownResync(session, node, say, shadowEnv, composes)
shadow := selfcompose.NewShadow(shadowEnv,
@@ -659,33 +658,16 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus
go resync.take(ctx, answers)
})
go shadow.Keep(run, link.Backoff) // this run's alone: a switch begins a run, and its shadow, again
composer, why := selfComposer(linked, session, node, say, hooks)
if linked.Err() != nil {
// The link ended before the node knew whose compositions to apply: it chose neither, and waits for the next.
stopRun()
cancel()
continue
}
if composer != nil {
machinesNow.Store(composer) // the machines a failing network part points at, by the placements (issue 774)
composerNow.Store(composer) // what the verb compose asks a pass of, this run's alone
composes.set(delivery.ComposesOwn)
env.Composed, env.SelfComposed, env.Trusted, env.Policy = composer, true, nil, composer.Policy
env.HoldsData = composer.HoldsData // what runs from the kept declaration says Data as its manifest does
session.NoteComposes(linked, delivery.ComposesOwn, why)
// And once the catalogue no longer places it, the agent begins again on the controller's compositions: the
// placement key deleted ends this run, the way back (issue 1126, the seed's revert).
go endWhenUnplaced(run, session, node, stopRun, say)
} else {
composes.set(delivery.ComposesController)
go resync.again(run)
session.NoteComposes(linked, delivery.ComposesController, why)
// And once the catalogue places it, the agent begins again composing its own: the placement key appearing ends
// this run, so a node does not wait for a restart to follow the seed of its placement (ADR 0549 §4).
go endWhenPlaced(run, session, node, stopRun, say, func(reason string) {
session.NoteComposes(linked, delivery.ComposesController, why+"; "+reason)
})
}
// Its own composer, always (issue 1126, A7): a composition made elsewhere (the controller's, in the node's
// composition bucket) is never applied. While the seat's placement holds no key for this node it composes
// nothing and keeps running what it runs; a placement it cannot watch stops the agent, said, and it begins again.
composer := selfComposer(session, node, hooks, say)
machinesNow.Store(composer) // the machines a failing network part points at, by the placements (issue 774)
composerNow.Store(composer) // what the verb compose asks a pass of, this run's alone
composes.set(delivery.ComposesOwn)
env.Composed, env.SelfComposed, env.Policy = composer, true, composer.Policy
env.HoldsData = composer.HoldsData // what runs from the kept declaration says Data as its manifest does
session.NoteComposes(linked, delivery.ComposesOwn, "it composes its own modules, and applies no composition made elsewhere")
ag = agent.New(env)
own.set(ag)
current.set(ag)
@@ -1001,8 +983,6 @@ func applyContext(ctx context.Context) (context.Context, context.CancelFunc) {
return context.WithTimeout(context.WithoutCancel(ctx), applyBound)
}
// selfComposer is the node's own composer when the catalogue places the node (its key `placement.<node>` in the
// indexer's state), and nil, said once, while it does not or the key cannot be read.
// composerHooks are what the composer reads of the agent and the machine (selfcompose.Env Running, Unseal, Refused).
type composerHooks struct {
running func() []delivery.Composed
@@ -1014,12 +994,9 @@ type composerHooks struct {
certify func(ctx context.Context) error
}
func selfComposer(ctx context.Context, session *link.Session, node string, say func(string), hooks composerHooks) (*selfcompose.Composer, string) {
key, err := delivery.PlacementNodeKey(node)
if err != nil {
say("this node applies the controller's compositions: " + err.Error())
return nil, err.Error()
}
// selfComposer is the node's own composer: it composes what the seat's placement places on the node, and nothing while
// the placement holds no key for it (issue 1126).
func selfComposer(session *link.Session, node string, hooks composerHooks, say func(string)) *selfcompose.Composer {
env := composerEnv(session, node, say, hooks)
// And it keeps each placed module's data items in its own state data, which the node's disk-load watches (novox/hq
// ADR 0577 clause 1); the shadow composes for candidates and keeps none.
@@ -1027,27 +1004,7 @@ func selfComposer(ctx context.Context, session *link.Session, node string, say f
// And the packages its compositions declare in its own key of the state packages, which pacman reads (novox/hq ADR
// 0565 §6).
env.Packages = link.EnginePackages{Session: session}
unreadable := ""
placed, err := selfcompose.Placed(ctx, env.Get, node, link.Backoff, func(m string) {
if strings.Contains(m, "cannot be read") {
unreadable = m
}
say(m)
}, selfcompose.ComposedItsOwn(ctx, env.Get, node))
if err != nil {
return nil, "its placement could not be read before the link ended: " + err.Error()
}
if !placed {
why := "the seat's placement state holds no key for it"
if unreadable != "" {
why = "its placement cannot be read, and it never composed its own: " + unreadable
}
say("this node applies the controller's compositions: " + why)
return nil, why
}
why := "the seat's placement state holds its key (" + key + ")"
say("this node composes its own modules: " + why)
return selfcompose.New(env), why
return selfcompose.New(env)
}
// composerEnv is what the node's composers read of the bus, the vault and the agent: its own composer and its answer
@@ -1084,95 +1041,3 @@ func composerEnv(session *link.Session, node string, say func(string), hooks com
Running: hooks.running, Unseal: hooks.unseal, Refused: hooks.refused, Held: hooks.held,
VaultAuthority: hooks.vaultAuthority, Certify: hooks.certify}
}
// placementKeysOf is a node's node key and the pattern of the keys that say it is placed: its node key and its legacy
// key `placement.<node>`, read while the holder may not have converted it (research 072 row 37). Its module keys say
// nothing of whether it is placed.
func placementKeysOf(node string) (key, keys string, err error) {
key, err = delivery.PlacementNodeKey(node)
if err != nil {
return "", "", err
}
legacy, _ := delivery.LegacyPlacementKey(node)
return key, link.WatchPatterns(key, legacy), nil
}
// anyHeld says some key holds a value.
func anyHeld(held map[string]bool) bool {
for _, h := range held {
if h {
return true
}
}
return false
}
// endWhenUnplaced calls stop once this node's placement key is deleted from the seat's placement (after the watch
// caught up, or gone already), so the agent begins again and applies the controller's compositions, as a node the
// catalogue does not place does from its start: the way back (issue 1126, the seed's revert). A watch that cannot be
// made is said; the node then returns at its next start.
func endWhenUnplaced(ctx context.Context, session *link.Session, node string, stop func(), say link.Announce) {
key, keys, err := placementKeysOf(node)
if err != nil {
return
}
updates, err := bucketWatch{session, delivery.PlacementBucket}.Watch(ctx, keys)
if err != nil {
say("whether this node's key in the seat's placement state still exists is not followed (" + err.Error() + "): it returns to the controller's compositions at its next start")
return
}
// Its node key or, before the holder converted it, its legacy key: placed while either holds a value (research
// 072 row 37); the holder deletes the legacy key before the node key, so the node key's delete is the last.
held, caught := map[string]bool{}, false
for u := range updates {
if u.Caught {
caught = true
} else {
held[u.Key] = !u.Deleted && len(u.Value) > 0
}
if caught && !anyHeld(held) {
say("this node's key in the seat's placement state is gone (" + key + "): the node agent begins again, applying the controller's compositions")
stop()
return
}
}
if ctx.Err() == nil {
say("the watch of " + key + " ended while the link stays: whether this node's key in the seat's placement state still exists is followed again at the node agent's next start")
}
}
// endWhenPlaced calls stop once this node's placement key appears in the seat's placement (a value after the watch
// caught up, or one already there), so the agent begins again and composes its own. A watch that cannot be made is said;
// the node then switches at its next start.
func endWhenPlaced(ctx context.Context, session *link.Session, node string, stop func(), say link.Announce, note func(string)) {
key, keys, err := placementKeysOf(node)
if err != nil {
return
}
// The seat's placement state is made when its holder starts; the delivery brings the grants part and the holder first,
// so it is normally there. A node-engine that starts before it waits for the bus's word that the bucket was created
// (no timer), says so once in the journal and in the node's own have, where the watchman reads it, and stays on the
// controller's compositions until its key is written. A watch the account may not make is said the same way, and the
// node stays until its next start.
updates, err := bucketWatch{session, delivery.PlacementBucket}.WatchAwait(ctx, keys, func(err error) {
why := "this node's key in the seat's placement state is waited for, the state is not on the bus yet (" + err.Error() + "): it stays on the controller's compositions and is watched when the bus says the state was created"
say(why)
note(why)
})
if err != nil {
why := "this node's key in the seat's placement state cannot be followed (" + err.Error() + "): it stays on the controller's compositions until its next start"
say(why)
note(why)
return
}
for u := range updates {
if !u.Caught && !u.Deleted && len(u.Value) > 0 {
say("this node's key in the seat's placement state is written now (" + u.Key + "): the node agent begins again, composing its own modules")
stop()
return
}
}
if ctx.Err() == nil {
say("the watch of " + key + " ended while the link stays: whether this node's key in the seat's placement state is written is followed again at the node agent's next start")
}
}
cmd/mesh-host/nodeagent_integration_test.go
+16 -231
View File
@@ -21,7 +21,6 @@ import (
"github.com/novox/mesh-host/internal/agent"
"github.com/novox/mesh-host/internal/link"
"github.com/novox/mesh-host/internal/selfcompose"
"github.com/novox/mesh-host/internal/store"
)
// The node agent whole, against a real bus with JetStream started in the test: the wants bucket, the node's composition
@@ -160,7 +159,7 @@ func (b *itBed) start(trustOperator bool) (stop func()) {
go func() {
defer close(done)
controller := b.controll.Public().(ed25519.PublicKey)
nodeAgent(ctx, b.opts, controller, newMeshTrust(controller), b.session, nil, b.say, func() {},
nodeAgent(ctx, b.opts, controller, newMeshTrust(), b.session, nil, b.say, func() {},
&agentNow{}, func(context.Context, *agent.Agent) {})
}()
return func() { cancel(); <-done }
@@ -237,126 +236,6 @@ func within(t *testing.T, d time.Duration, what string, ok func() bool) {
}
}
// R6-A AC1: given a running agent on a bus, when a want naming a digest and that composition are written, the module is
// applied within one second and its have names the fingerprint and the digest; a composition whose digest no want names
// is not applied.
func TestNatsAWantAndItsCompositionAreAppliedWithinASecond(t *testing.T) {
t.Run("1957: an engine start starts a declared-running unit", testNatsAnEngineStartStartsADeclaredRunningUnitTheHaveSaysRuns)
b := newItBed(t)
stop := b.start(false)
defer stop()
time.Sleep(300 * time.Millisecond) // the watches are up
fp := fpIt("a")
dig := b.compose("web", fp, "one\n", false, b.controll)
time.Sleep(300 * time.Millisecond)
if b.file("web") != "" {
t.Fatal("a composition no want names was applied")
}
began := time.Now()
b.want("web", fp, dig, "")
within(t, time.Second, "the module was not applied within one second of its want", func() bool { return b.file("web") == "one\n" })
t.Logf("applied %s after the want", time.Since(began))
within(t, 2*time.Second, "the have does not name the fingerprint and the digest", func() bool {
h := b.have("web")
return h.FP == fp && h.Composed == dig && h.Phase == delivery.PhaseRunning
})
// A want naming a digest the composition at its fingerprint does not have applies nothing.
fp2 := fpIt("b")
b.compose("web", fp2, "two\n", false, b.controll)
b.want("web", fp2, "sha256:"+strings.Repeat("0", 64), "")
time.Sleep(300 * time.Millisecond)
if b.file("web") != "one\n" {
t.Fatalf("a want naming another digest changed the module: %q", b.file("web"))
}
}
// R6-A AC5: a data module that ran before asks its backup before any resource changes; a backup that fails holds the
// module (nothing applied, the have says held with the backup's words); when its want then says skip, it is applied.
func TestNatsADataModuleAsksItsBackupFirstAndIsHeldWhenItFails(t *testing.T) {
b := newItBed(t)
var mu sync.Mutex
var asked []string
var onDiskWhenAsked string
if _, err := b.conn.Subscribe("mesh.seat.node-backup.tool.now."+itNode, func(m *nats.Msg) {
mu.Lock()
asked = append(asked, "now")
onDiskWhenAsked = b.file("db")
mu.Unlock()
_ = m.Respond([]byte(`{"result":{"started":"db"}}`))
// The holder says the backup ended on its event.
ev, _ := json.Marshal(map[string]any{"node": itNode, "module": "db", "ok": false, "at": time.Now().UTC(),
"error": "restic: the repository is locked"})
_ = b.conn.Publish(backedUpEvent, ev)
}); err != nil {
t.Fatal(err)
}
stop := b.start(false)
defer stop()
time.Sleep(300 * time.Millisecond)
fp1, fp2 := fpIt("a"), fpIt("b")
b.want("db", fp1, b.compose("db", fp1, "v1\n", true, b.controll), "")
within(t, 3*time.Second, "the first composition was not applied", func() bool { return b.file("db") == "v1\n" })
mu.Lock()
if len(asked) != 0 {
t.Fatalf("a module that never ran was backed up: %v", asked)
}
mu.Unlock()
dig2 := b.compose("db", fp2, "v2\n", true, b.controll)
b.want("db", fp2, dig2, "")
within(t, 3*time.Second, "the module was not held behind the failed backup", func() bool {
h := b.have("db")
return h.Phase == delivery.PhaseHeld && strings.Contains(h.Detail, "repository is locked")
})
mu.Lock()
if len(asked) != 1 || onDiskWhenAsked != "v1\n" {
t.Fatalf("the backup was asked %v, with %q on disk: it is asked before any resource changes", asked, onDiskWhenAsked)
}
mu.Unlock()
if b.file("db") != "v1\n" {
t.Fatalf("something was applied behind a failed backup: %q", b.file("db"))
}
if h := b.have("db"); h.FP != fp1 {
t.Fatalf("a held module should still name what it runs: %+v", h)
}
b.want("db", fp2, dig2, delivery.BackupSkip)
within(t, 3*time.Second, "the module was not applied once its want said skip", func() bool { return b.file("db") == "v2\n" })
within(t, 2*time.Second, "the have does not say running at the new fingerprint", func() bool {
h := b.have("db")
return h.Phase == delivery.PhaseRunning && h.FP == fp2 && h.Composed == dig2
})
}
// R6-A AC9, the hand path end to end: with no controller running, a composition signed by the operator's key and a want
// naming it are applied and the have written; the same signed by a key nobody trusts is not.
func TestNatsTheHandPathAppliesACompositionTheOperatorSigned(t *testing.T) {
b := newItBed(t)
stop := b.start(true)
defer stop()
time.Sleep(300 * time.Millisecond)
_, stranger, _ := ed25519.GenerateKey(nil)
fpS := fpIt("c")
b.want("evil", fpS, b.compose("evil", fpS, "no\n", false, stranger), "")
time.Sleep(300 * time.Millisecond)
if b.file("evil") != "" {
t.Fatal("a composition signed by a key nobody trusts was applied")
}
fp := fpIt("d")
dig := b.compose("rescue", fp, "back\n", false, b.operator)
b.want("rescue", fp, dig, "")
within(t, 2*time.Second, "the operator's composition was not applied", func() bool { return b.file("rescue") == "back\n" })
within(t, 2*time.Second, "the have was not written", func() bool {
h := b.have("rescue")
return h.FP == fp && h.Composed == dig && h.Phase == delivery.PhaseRunning
})
}
// Issue 1126 (ADR 0547 §2, the milestone's H8): the node agent started whole against a real bus, once the catalogue
// places its node (the placement written after it started), composes a module from its want, the indexer's manifest, its record and its mesh-config value, and applies it,
// with no key in the controller's compositions; a changed config value is composed again on its event and applied as a
@@ -478,9 +357,10 @@ func h8once(t *testing.T, b *itBed, resolved, candidates nats.KeyValue, place []
}
}
// h8back is the way back (issue 1126, the seed's revert): a candidate deleting the node's placement is compared with
// the controller's compositions at its wants (here none: refused, naming web), and the placement key deleted returns the agent to them without a
// restart, said in the journal and the node's have, keeping what runs.
// h8back is A7 of issue 1126: no composition made elsewhere is applied. A candidate deleting the node's placement is
// refused, never answered as a return to the controller's compositions; the placement key deleted leaves the agent on
// its own composer, composing nothing and keeping what runs, said in the journal, the node's have still saying it
// composes its own.
func h8back(t *testing.T, b *itBed, placement, resolved, candidates nats.KeyValue, hkey string) {
t.Helper()
node := func() delivery.HaveNode {
@@ -502,65 +382,38 @@ func h8back(t *testing.T, b *itBed, placement, resolved, candidates nats.KeyValu
if _, err := candidates.Put(ckey, craw); err != nil {
t.Fatal(err)
}
// The controller holds no composition at web's want (the node composed its own): returning would leave web with
// nothing to apply, and the answer says so rather than "same" (review of the shadow, round 1).
within(t, 10*time.Second, "the node did not refuse returning to compositions the controller does not hold", func() bool {
within(t, 10*time.Second, "the node did not refuse a candidate taking its placement away", func() bool {
a := node().Candidates["revert-x"]
return a.Proposal == "revert-x" && a.Returns == "" && a.Verdict == delivery.CandidateRefused &&
strings.Contains(a.Refused, "web: the controller holds no composition")
strings.Contains(a.Refused, "composes nothing")
})
ran := b.have("web").Composed
pkey, _ := delivery.PlacementNodeKey(itNode)
// A pass in flight at the switch (issue 1126, 2026-10-28: the last pass of the own run, failing on its run's end,
// marked the next agent's modules failed): a config value changed starts one, and the key goes at once.
// A pass in flight when the key goes (issue 1126, 2026-10-28): a config value changed starts one, and the key goes at once.
if _, err := resolved.Put(itNode+".web.greeting", []byte(`{"value":"salut","source":"module"}`)); err != nil {
t.Fatal(err)
}
if err := placement.Delete(pkey); err != nil {
t.Fatal(err)
}
stopWatch := make(chan struct{})
marked := make(chan Have1126, 1)
go func() {
for {
select {
case <-stopWatch:
return
case <-time.After(20 * time.Millisecond):
}
if h := b.have("web"); h.Phase == delivery.PhaseFailed {
select {
case marked <- Have1126{h.Phase, h.Detail}:
default:
}
return
}
}
}()
defer close(stopWatch)
within(t, 15*time.Second, "the node did not return to the controller's compositions when its placement went", func() bool {
within(t, 15*time.Second, "the node did not say it composes nothing once its placement went", func() bool {
b.mu.Lock()
said := strings.Join(b.said, "\n")
b.mu.Unlock()
return strings.Contains(said, "key in the seat's placement state is gone") && node().Composes == delivery.ComposesController
return strings.Contains(said, "key in the seat's placement state is gone: it composes nothing of its own")
})
time.Sleep(3 * time.Second) // what a pass in flight would mark lands within it
select {
case h := <-marked:
t.Errorf("a pass in flight at the switch marked web failed: %+v", h)
default:
b.mu.Lock()
said := strings.Join(b.said, "\n")
b.mu.Unlock()
if n := node(); n.Composes != delivery.ComposesOwn || strings.Contains(said, "applying the controller's compositions") {
t.Errorf("without its placement the node turned to the controller's compositions: %q", n.Composes)
}
if h := b.have("web"); h.Composed != ran || h.Phase == delivery.PhaseFailed || b.file("web") != "greeting=salut\n" {
t.Errorf("returning to the controller's compositions moved what runs: %+v, %q", h, b.file("web"))
t.Errorf("without its placement what runs moved: %+v, %q", h, b.file("web"))
}
}
// Have1126 is what H8's way back watches of web's have.
type Have1126 struct {
Phase delivery.Phase
Detail string
}
// h8state is what h8 set up that the rest of H8 writes to.
type h8state struct {
placement, resolved, candidates nats.KeyValue
@@ -730,71 +583,3 @@ func h8rest(t *testing.T, b *itBed, resolved, candidates nats.KeyValue, hkey str
t.Errorf("a candidate moved what runs: %+v, %q", h, b.file("web"))
}
}
// 1957/AC1 (integration; ADR 0568 §1): a node agent starting on a bus that holds a `running` have for a module at its want,
// with a service manager (a systemctl on the PATH) that shows the module's unit inactive, starts the unit once and writes
// the have running; the same start with the unit active applies nothing.
func testNatsAnEngineStartStartsADeclaredRunningUnitTheHaveSaysRuns(t *testing.T) {
for _, active := range []bool{false, true} {
b := newItBed(t)
bin := t.TempDir()
stateFile, calls := filepath.Join(bin, "state"), filepath.Join(bin, "calls")
if active {
os.WriteFile(stateFile, []byte("active"), 0o644)
} else {
os.WriteFile(stateFile, []byte("inactive"), 0o644)
}
script := "#!/bin/sh\necho \"$@\" >> " + calls + "\ncase \"$1\" in\n" +
"show) printf 'LoadState=loaded\\nActiveState=%s\\nType=simple\\n' \"$(cat " + stateFile + ")\" ;;\n" +
"start) printf active > " + stateFile + " ;;\nis-enabled) echo enabled ;;\nesac\nexit 0\n"
if err := os.WriteFile(filepath.Join(bin, "systemctl"), []byte(script), 0o755); err != nil {
t.Fatal(err)
}
t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH"))
// The engine's own record of the unit, as a previous run left it, and the have it wrote then.
res, _ := json.Marshal(map[string]any{"id": "web.unit", "type": "service", "unit": "web-it.service", "state": "running", "module": "web"})
var known store.State
known.Record(store.Applied{ID: "web.unit", Type: "service", Target: "web-it.service"})
if err := store.Save(b.opts.state, known); err != nil {
t.Fatal(err)
}
fp := fpIt("a")
c := delivery.Composed{Node: itNode, Module: "web", FP: fp, Resources: []json.RawMessage{res}}
signed, _ := delivery.SignComposed(c, b.controll)
raw, _ := delivery.EncodeComposed(signed)
dig, _ := delivery.ComposedDigestOf(signed)
key, _ := delivery.ComposedKey(itNode, "web", fp)
if _, err := b.composed.Put(key, raw); err != nil {
t.Fatal(err)
}
b.want("web", fp, dig, "")
hkey, _ := delivery.HaveKey(itNode, "web")
have, err := delivery.EncodeHave(delivery.Have{FP: fp, Composed: dig, Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown})
if err != nil {
t.Fatal(err)
}
if _, err := b.haves.Put(hkey, have); err != nil {
t.Fatal(err)
}
stop := b.start(false)
within(t, 5*time.Second, "the unit was not started and the have not running", func() bool {
got, _ := os.ReadFile(stateFile)
h := b.have("web")
return string(got) == "active" && h.Phase == delivery.PhaseRunning && h.Composed == dig
})
time.Sleep(300 * time.Millisecond)
stop()
starts := 0
log, _ := os.ReadFile(calls)
for _, l := range strings.Split(string(log), "\n") {
if strings.HasPrefix(l, "start") {
starts++
}
}
if want := map[bool]int{false: 1, true: 0}[active]; starts != want {
t.Fatalf("unit active=%v: started %d times, want %d", active, starts, want)
}
}
}
cmd/mesh-host/trust.go
+20 -80
View File
@@ -13,7 +13,6 @@ import (
"sync"
"time"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/overlay"
@@ -21,30 +20,25 @@ import (
"github.com/novox/mesh-host/internal/link"
)
// **What this engine trusts a mesh signature from** (novox/hq ADR 0550 §2; design 47's L3 item d). Two signers: the
// controller's key, which the node was enrolled with (its membership's signer), and the vault's key, read from the vault's
// state `signers`, which only the vault may write: the bus grants are its anchor, as the wants' and the placement's (the
// lead's design limit, 2026-10-28). The controller's key is trusted until its lease is handed away, heard on the lease key
// (buscompose.LeaseHandedAway) and never on a clock; from then the vault's alone. What it trusts now the node states in its
// record (NodeRecord.Trusts), so the gate before the lease write reads every node's from the bus.
// **What this engine trusts a mesh signature from** (novox/hq ADR 0550 §2; design 47's L3 item d; issue 1126): the
// vault's key alone, read from the vault's state `signers`, which only the vault may write: the bus grants are its
// anchor, as the wants' and the placement's. The controller's key, which the node was enrolled with (its membership's
// signer), is trusted never: the controller ended by its unassignment (the operator, 2026-11-02) and its lease is never
// handed, so no gate waits on it. What it trusts now the node states in its record (NodeRecord.Trusts).
type meshTrust struct {
mu sync.Mutex
controller ed25519.PublicKey
handedAway bool
vault ed25519.PublicKey
authority string
mu sync.Mutex
vault ed25519.PublicKey
authority string
// changed is told when what it trusts changed, so the record is written again.
changed func()
}
// newMeshTrust trusts the controller's key alone until the vault's is read.
func newMeshTrust(controller ed25519.PublicKey) *meshTrust {
t := &meshTrust{}
if len(controller) == ed25519.PublicKeySize {
t.controller = controller
}
return t
}
// newMeshTrust trusts nothing until the vault's key is read.
func newMeshTrust() *meshTrust { return &meshTrust{} }
// noControllerKey is the controller's key as this engine trusts it: none. A link told it refuses every ask signed as
// the controller's, where a link told nothing would verify it with the membership's signer (link.HandOverCheck).
func noControllerKey() ed25519.PublicKey { return nil }
// onChange names who is told when what it trusts changed.
func (t *meshTrust) onChange(f func()) {
@@ -53,38 +47,11 @@ func (t *meshTrust) onChange(f func()) {
t.mu.Unlock()
}
// Controller is the controller's key while the controller holds its lease, and nil once the lease is handed away.
func (t *meshTrust) Controller() ed25519.PublicKey {
if t == nil {
return nil
}
t.mu.Lock()
defer t.mu.Unlock()
if t.handedAway {
return nil
}
return t.controller
}
// HandedAway says the controller's lease was handed away, as heard on the lease key.
func (t *meshTrust) HandedAway() bool {
if t == nil {
return false
}
t.mu.Lock()
defer t.mu.Unlock()
return t.handedAway
}
// Signers are the keys a mesh signature is verified against now: the controller's while it holds the lease, and the
// vault's once read.
// Signers are the keys a mesh signature is verified against now: the vault's once read, and none before.
func (t *meshTrust) Signers() []delivery.Signer {
t.mu.Lock()
defer t.mu.Unlock()
var out []delivery.Signer
if t.controller != nil && !t.handedAway {
out = append(out, delivery.Signer{Name: delivery.SignerController, Key: t.controller})
}
if t.vault != nil {
out = append(out, delivery.Signer{Name: delivery.SignerVault, Key: t.vault})
}
@@ -165,21 +132,11 @@ func (t *meshTrust) heardSigner(key string, value []byte, deleted bool, say link
}
}
// heardLease takes the controller's lease key as read: once handed away to a holder that is no controller, the
// controller's key is trusted no more. A lease that cannot be read changes nothing.
func (t *meshTrust) heardLease(raw []byte, found bool, readErr error) {
if readErr != nil {
return
}
handed, _ := buscompose.LeaseHandedAway(raw, found, nil)
t.change(func() { t.handedAway = handed })
}
// trustFollowAgain bounds the wait before a watch that could not be made is tried again on the same link: the vault may
// not serve `sign` yet, and until it does this node is not granted the reads.
var trustFollowAgain = 2 * time.Minute
// followTrust follows the vault's `signers` and the controller's lease on every link, as watches (never a poll), for as
// followTrust follows the vault's `signers` on every link, as watches (never a poll), for as
// long as ctx lasts. A watch that cannot be made is said once per reason and tried again.
func followTrust(ctx context.Context, session *link.Session, trust *meshTrust, say link.Announce) {
said := ""
@@ -198,33 +155,16 @@ func followTrust(ctx context.Context, session *link.Session, trust *meshTrust, s
}
watching, stop := context.WithCancel(linked)
signers, serr := session.KVWatch(watching, delivery.SignersBucket, ">")
lease, lerr := session.KVWatch(watching, buscompose.LeaseBucket, buscompose.ControllerLeaseKey)
if serr != nil || lerr != nil {
if serr != nil {
stop()
sayOnce(fmt.Sprintf("what this node trusts a mesh signature from is not followed yet (the vault's signers: %v; "+
"the controller's lease: %v): it trusts what it last read, and looks again in %s", serr, lerr, trustFollowAgain))
signers, lease = nil, nil
sayOnce(fmt.Sprintf("what this node trusts a mesh signature from is not followed yet (the vault's signers: %v): "+
"it trusts what it last read, and looks again in %s", serr, trustFollowAgain))
} else {
sayOnce("")
}
for signers != nil || lease != nil {
select {
case u, ok := <-signers:
if !ok {
signers = nil
continue
}
for u := range signers {
if !u.Caught {
trust.heardSigner(u.Key, u.Value, u.Deleted, say)
}
case u, ok := <-lease:
if !ok {
lease = nil
continue
}
if !u.Caught {
trust.heardLease(u.Value, !u.Deleted && len(u.Value) > 0, nil)
}
}
}
stop()
cmd/mesh-host/userlist.go
+4 -24
View File
@@ -20,8 +20,8 @@ import (
"github.com/novox/mesh-host/internal/userlist"
)
// The bridge of novox/hq ADR 0547 §8, wired: the bus node's engine writes the bus's user list alone from the newest
// composition of the bus module, under the apply's two locks as a filled file is written (filling.go), and says what
// The bridge of novox/hq ADR 0547 §8, wired: the bus node's engine writes the grants part's user list alone into the file
// the bus module's composition declares, under the apply's two locks as a filled file is written (filling.go), and says what
// it wrote in the bus module's have.
// userListFollower is the follower of the bus's user list for one link: the node's composition bucket, the keys the
@@ -32,10 +32,7 @@ func userListFollower(opts options, session *link.Session, composedBucket string
Node: session.Membership.Node,
Composed: bucketWatch{session, composedBucket},
Trusted: trusted,
Resolve: func(ctx context.Context, raw json.RawMessage) ([]byte, error) {
return resolveUserList(session.WithLink(ctx), raw)
},
OnDisk: userListOnDisk,
OnDisk: userListOnDisk,
Write: func(ctx context.Context, module string, raw json.RawMessage, body []byte) (string, bool, error) {
digest, changed, err := writeUserListAlone(session.WithLink(ctx), opts, raw, body, say)
if changed && wrote != nil {
@@ -75,10 +72,7 @@ func userListFollower(opts options, session *link.Session, composedBucket string
Kept: func() ([]byte, uint64) { return keptGrantsKey(opts) },
Keep: func(value []byte, rev uint64) { keepGrantsKey(opts, value, rev, say) },
Backoff: link.Backoff,
// The vault's key from its answer to node-key: the loaded list's hashes are handed sealed to it (ADR 0547
// clause 9).
VaultKey: vaultKey.get,
Say: announceOr(say),
Say: announceOr(say),
})
}
@@ -98,20 +92,6 @@ func userListFile(raw []byte) (*declaration.File, error) {
return f, nil
}
// resolveUserList is the list a user-list resource holds: its content, or the object its digest names in this node's
// file bucket, fetched and verified (apply.ResolveFileReferences, ADR 0486 §1).
func resolveUserList(ctx context.Context, raw []byte) ([]byte, error) {
f, err := userListFile(raw)
if err != nil {
return nil, err
}
d := &declaration.Declaration{Version: declaration.Version, Resources: []declaration.Resource{f}}
if err := apply.ResolveFileReferences(ctx, d, link.FilesFrom(ctx)); err != nil {
return nil, fmt.Errorf("the user list could not be had by its digest: %w", err)
}
return []byte(d.Resources[0].(*declaration.File).Content), nil
}
// userListOnDisk is what the user list's file holds on this machine now; nil when there is none.
func userListOnDisk(raw json.RawMessage) ([]byte, error) {
f, err := userListFile(raw)
internal/selfcompose/selfcompose.go
+2 -70
View File
@@ -798,8 +798,8 @@ func (c *Composer) compose(ctx context.Context) []agent.Update {
r := reader{c: c}
c.mu.Unlock()
if !placed {
// The catalogue no longer places it: the agent returns to the controller's compositions (endWhenUnplaced), and
// this is no refusal of what it runs (issue 1126, the seed's revert).
// The catalogue does not place it: it composes nothing, applies no composition made elsewhere, and this is no
// refusal of what it runs (issue 1126).
c.sayOnce("this node's key in the seat's placement state is gone: it composes nothing of its own, and keeps running what it runs")
c.paid() // nothing of its own to compose, so no pass is owed and no timer armed again
return nil
@@ -1496,74 +1496,6 @@ func (r reader) Credentials(ctx context.Context, node string, asConsumer, asProv
return creds, issued, nil
}
// Placed reads whether the catalogue places node now, for the node agent choosing whose compositions it applies at
// each start: true when its placement key holds a value, false only when a read succeeds and finds no key or a delete
// marker. A read that fails chooses neither (review of the shadow, round 1: a timeout or a bus not ready at a restart
// flipped a self-composing node to the controller's compositions for a whole run): it is tried again after wait(try),
// each distinct failure said once, until ctx ends, which is then its error.
//
// switched says the node composed from its own placement key before (its have says it composes its own,
// delivery.ComposesOwn, and that is kept on the bus, so it survives a restart). Only a node that never switched takes a
// read that fails (a timeout, or a state its account may not read yet, as the seat's before its grant arrives) as "not
// placed": it stays on the controller's compositions, says why once, and switches on a positive read of its key (the
// watch endWhenPlaced keeps). A node that switched never goes back on a failed read, however long it lasts.
func Placed(ctx context.Context, get func(ctx context.Context, bucket, key string) ([]byte, error), node string,
wait func(try int) time.Duration, say func(string), switched bool) (bool, error) {
key, err := delivery.PlacementNodeKey(node)
if err != nil {
return false, err
}
legacy, _ := delivery.LegacyPlacementKey(node)
said := ""
for try := 1; ; try++ {
// Its node key, or, before the seat's holder converted it, its legacy key (research 072 row 37).
raw, err := get(ctx, delivery.PlacementBucket, key)
if err == nil && len(raw) == 0 {
raw, err = get(ctx, delivery.PlacementBucket, legacy)
}
if errors.Is(err, nats.ErrBucketNotFound) {
return false, nil // the seat has written no placement state yet: nothing places it, read and known
}
if err == nil {
return len(raw) > 0, nil
}
if ctx.Err() != nil {
return false, ctx.Err()
}
if !switched {
say(fmt.Sprintf("this node's key in the seat's placement state cannot be read (%s): it has never composed its own, so it applies the controller's compositions and switches when its key is read: %v", key, err))
return false, nil
}
if why := err.Error(); why != said {
say(fmt.Sprintf("this node's key in the seat's placement state cannot be read (%s): the node agent waits, choosing neither its own compositions nor the controller's: %v", key, err))
said = why
}
select {
case <-ctx.Done():
return false, ctx.Err()
case <-time.After(wait(try)):
}
}
}
// ComposedItsOwn says whether the node's own have, kept on the bus, says it composed from its placement key at its last
// start. A have that cannot be read says yes: the safe side, a node waits and does not fall back.
func ComposedItsOwn(ctx context.Context, get func(ctx context.Context, bucket, key string) ([]byte, error), node string) bool {
key, err := delivery.HaveNodeKey(node)
if err != nil {
return true
}
raw, err := get(ctx, delivery.HaveBucket, key)
if err != nil {
return true
}
if len(raw) == 0 {
return false
}
n, err := delivery.DecodeHaveNode(raw)
return err != nil || n.Composes == delivery.ComposesOwn
}
// composeAgain are the waits before a pass that met a bus not answering is tried again.
var composeAgain = []time.Duration{time.Second, 2 * time.Second, 4 * time.Second}
internal/selfcompose/selfcompose_test.go
+17 -94
View File
@@ -1162,9 +1162,11 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
return a.Verdict == delivery.CandidateSame && a.Returns == "" && a.Revision == 8 && len(a.Removed) == 0 &&
fmt.Sprint(a.Same) == "[lemurs node]"
},
// Issue 1126, A7: a candidate deleting the node's placement is refused, never answered as a return to the
// controller's compositions: no composition made elsewhere is applied, so the node would compose nothing.
"deletes": func(a delivery.CandidateAnswer) bool {
return a.Verdict == delivery.CandidateSame && a.Returns == delivery.ReturnsToController && a.Revision == 7 &&
len(a.Removed) == 0 && fmt.Sprint(a.Same) == "[lemurs node]"
return a.Verdict == delivery.CandidateRefused && a.Returns == "" && a.Revision == 7 &&
strings.Contains(a.Refused, "composes nothing")
},
} {
if !want(answers[group]) {
@@ -1286,61 +1288,21 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
return !ok && len(a) == 4
})
// Whose compositions a node applies at a start (review of the shadow, round 1): only a read that succeeds chooses; a
// failing read is tried again, said once, and chooses neither, even when the link ends.
for name, row := range map[string]struct {
answers []error // per try; nil then the value
value []byte
end bool
placed bool
tries int
quiet bool // read and known: nothing to say
// switched: the node's have says it composed its own before (it never falls back on a failed read).
switched bool
}{
"switched: placed after two failures": {answers: []error{fmt.Errorf("nats: timeout"), fmt.Errorf("nats: timeout")}, value: []byte(`{}`), placed: true, tries: 3, switched: true},
"never switched: a timeout is not placed, said once": {answers: []error{fmt.Errorf("nats: timeout")}, placed: false, tries: 1},
"no key": {placed: false, tries: 2}, // its node key, then its legacy key (research 072 row 37)
"no placement state yet": {answers: []error{fmt.Errorf("the bucket %s: %w", delivery.PlacementBucket, nats.ErrBucketNotFound)}, placed: false, tries: 1, quiet: true},
"switched: failing until the link ends": {answers: []error{fmt.Errorf("jetstream not ready"), fmt.Errorf("jetstream not ready"),
fmt.Errorf("jetstream not ready")}, end: true, tries: 3, switched: true},
} {
lctx, lcancel := context.WithCancel(ctx)
tries := 0
var said []string
get := func(context.Context, string, string) ([]byte, error) {
tries++
if tries <= len(row.answers) {
if row.end && tries == len(row.answers) {
lcancel()
}
return nil, row.answers[tries-1]
// Issue 1126, A7 AC1: a placement that cannot be read composes nothing and says why; the node never applies a
// composition made elsewhere instead. (A placement read without the node's key composes nothing either: above.)
{
refused := newFakeBus()
env := refused.env()
watch := env.Watch
env.Watch = func(ctx context.Context, bucket, pattern string) (<-chan agent.Update, error) {
if bucket == delivery.PlacementBucket {
return nil, fmt.Errorf("nats: Permissions Violation for Subscription to %s", bucket)
}
return row.value, nil
return watch(ctx, bucket, pattern)
}
placed, err := Placed(lctx, get, "n1", func(int) time.Duration { return time.Millisecond }, func(s string) { said = append(said, s) }, row.switched)
lcancel()
if placed != row.placed || tries != row.tries || (err != nil) != row.end || (len(row.answers) > 0 && !row.quiet) != (len(said) == 1) {
t.Errorf("%s: placed %v, err %v after %d reads, said %q", name, placed, err, tries, said)
}
}
// Whether the node composed its own before is read from its own have: own says yes, the controller's or no have says
// no, and a have that cannot be read says yes (the safe side: a node waits and does not fall back).
for name, row := range map[string]struct {
raw []byte
err error
want bool
}{
"own": {raw: []byte(`{"v":1,"composes":"own"}`), want: true},
"the controller's": {raw: []byte(`{"v":1,"composes":"controller"}`), want: false},
"no have yet": {want: false},
"a have that cannot be read": {err: fmt.Errorf("nats: timeout"), want: true},
"a have that is no have": {raw: []byte(`{`), want: true},
} {
get := func(context.Context, string, string) ([]byte, error) { return row.raw, row.err }
if got := ComposedItsOwn(ctx, get, "n1"); got != row.want {
t.Errorf("composed its own, %s: %v, want %v", name, got, row.want)
out, err := New(env).Watch(ctx, "")
if out != nil || err == nil || !strings.Contains(err.Error(), delivery.PlacementBucket) || !strings.Contains(err.Error(), "cannot be watched") {
t.Errorf("an unreadable placement: composed %v, said %v", out != nil, err)
}
}
@@ -1450,45 +1412,6 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
}
}
// The way back, row by row (review of the shadow, round 1): the controller's compositions at n1's wants compared
// with what runs; one differing is answered so; one the controller holds none of refuses, without returning.
var runsList []delivery.Composed
for _, c := range running {
runsList = append(runsList, c)
}
lemursKey, _ = delivery.ComposedKey("n1", "lemurs", fp)
changed := running[lemursKey]
changed.Resources = []json.RawMessage{json.RawMessage(`{"id":"lemurs.service","type":"service","module":"lemurs","unit":"lemurs.service","state":"stopped","boot":"disabled"}`)}
changedRaw, _ := delivery.EncodeComposed(changed)
for name, row := range map[string]struct {
lemurs []byte
check func(delivery.CandidateAnswer) bool
}{
"the controller's differs": {changedRaw, func(a delivery.CandidateAnswer) bool {
return a.Verdict == delivery.CandidateDiffers && a.Returns == delivery.ReturnsToController &&
len(a.Differs) == 1 && a.Differs[0].Module == "lemurs" && fmt.Sprint(a.Same) == "[node]"
}},
"the controller holds none": {nil, func(a delivery.CandidateAnswer) bool {
return a.Verdict == delivery.CandidateRefused && a.Returns == "" && strings.Contains(a.Refused, "lemurs: the controller holds no composition")
}},
} {
get := func(_ context.Context, bucket, key string) ([]byte, error) {
if key == lemursKey {
return row.lemurs, nil
}
readOnly.mu.Lock()
defer readOnly.mu.Unlock()
return readOnly.kv[bucket+"/"+key], nil
}
a := returning(ctx, get, "n1", map[string]delivery.Want{"lemurs": {FP: fp}, agent.NodeModule: {}}, runsList, "pro", 1)
if !row.check(a) {
t.Errorf("%s: %+v", name, a)
}
if _, err := delivery.EncodeHaveNode(delivery.HaveNode{Candidates: map[string]delivery.CandidateAnswer{"pro": a}}); err != nil {
t.Errorf("%s: the answer is not one a have holds: %v", name, err)
}
}
// Issue 1126, the seed's review: a module n1 runs inside its own composition (nats, at a build of the old form) is
// composed there again, not at the trunk's build as one added; lemurs at the build it runs, keeper (added) at the
// trunk's.
internal/selfcompose/shadow.go
+9 -73
View File
@@ -23,8 +23,7 @@ import (
//
// **Event-driven, like the Composer.** It watches every candidate key, the trunk's placement, every node's record, the
// indexer's modules, this node's config values, and this node's module haves (what it runs moved); a burst of changes
// answers once. It runs whether or not the node is placed yet: a node that applies the controller's compositions answers
// against what it runs from them. An answer is handed only when it changes, and a proposal's answer is dropped when its
// answers once. It runs whether or not the node is placed yet, against what it runs. An answer is handed only when it changes, and a proposal's answer is dropped when its
// keys go.
type Shadow struct {
c *Composer
@@ -33,8 +32,7 @@ type Shadow struct {
mu sync.Mutex
candidates map[string]map[string]candidateKey // by proposal, by node
answers map[string]delivery.CandidateAnswer
// wants are the node's wants by module (agent.NodeModule for its own): what a proposal deleting its placement returns
// it to, the controller's compositions at them.
// wants are the node's wants by module (agent.NodeModule for its own); a change of one answers again.
wants map[string]delivery.Want
// ran is, by module, the digest the module's have says runs: what runs, where the agent's list holds only the copy
// from the declaration kept before the switch (issue 1126, probe #532 on shanks).
@@ -161,7 +159,7 @@ func (s *Shadow) Run(ctx context.Context) error {
{ResolvedBucket, node + ".>", (*Composer).takeValue, nil},
// What this node runs moved: an apply ends in its module's have.
{delivery.HaveBucket, node + ".>", nil, nil},
// Its wants: a proposal deleting its placement returns it to the controller's compositions at them.
// Its wants.
{delivery.AssignerWantsBucket, "want." + node + ".>", nil, nil},
}
type tagged struct {
@@ -530,16 +528,12 @@ func (s *Shadow) answerProposal(ctx context.Context, proposal string) (delivery.
running = c.env.Running()
}
if p, ok := given[node]; ok && p == nil && !own.c.Reached {
// The proposal deletes this node's placement key: the node then returns to the controller's compositions, as one the
// catalogue does not place (endWhenUnplaced). Those at its wants are compared with what runs (issue 1126, the
// seed's revert, whose check read "would stop running node").
s.mu.Lock()
wants := make(map[string]delivery.Want, len(s.wants))
for m, w := range s.wants {
wants[m] = w
}
s.mu.Unlock()
return returning(ctx, c.get, node, wants, running, proposal, rev), true
// The proposal deletes this node's placement key: the node would compose nothing and keep running what it runs,
// since no composition made elsewhere is applied (issue 1126: the controller ended by its unassignment). Refused,
// so the catalogue does not take a node's placement away by a merge.
return delivery.CandidateAnswer{Proposal: proposal, Revision: rev, Verdict: delivery.CandidateRefused,
Refused: "the proposal takes this node's placement away: it then composes nothing, and no composition made " +
"elsewhere is applied, so what runs here would be followed by nothing"}, true
}
placement := given[node]
@@ -718,64 +712,6 @@ func (s *Shadow) takeWant(u agent.Update) bool {
return !had || old.FP != w.FP
}
// returning is the node's answer to a proposal deleting its placement: it returns to the controller's compositions, each
// read at the fingerprint its want names in the node's composition bucket and compared with what runs
// (delivery.CompareComposed). A want the controller holds no composition for, or one that cannot be read, refuses the
// answer, naming each: returning then would leave that module with nothing to apply.
func returning(ctx context.Context, get func(ctx context.Context, bucket, key string) ([]byte, error), node string,
wants map[string]delivery.Want, running []delivery.Composed, proposal string, rev uint64) delivery.CandidateAnswer {
a := delivery.CandidateAnswer{Proposal: proposal, Revision: rev}
bucket, err := delivery.ComposedBucketFor(node)
if err != nil {
a.Refused, a.Verdict = err.Error(), delivery.CandidateRefused
return a
}
modules := make([]string, 0, len(wants))
for m := range wants {
modules = append(modules, m)
}
sort.Strings(modules)
comps := map[string]delivery.Composed{}
var missing []string
for _, m := range modules {
var key string
if m == agent.NodeModule {
key, err = delivery.ComposedNodeKey(node)
} else {
key, err = delivery.ComposedKey(node, m, wants[m].FP)
}
if err != nil {
missing = append(missing, m+": "+err.Error())
continue
}
raw, err := get(ctx, bucket, key)
switch {
case err != nil:
missing = append(missing, fmt.Sprintf("%s: the controller's composition %s cannot be read: %v", m, key, err))
continue
case raw == nil:
missing = append(missing, fmt.Sprintf("%s: the controller holds no composition %s, which its want names", m, key))
continue
}
comp, err := delivery.DecodeComposed(raw)
if err != nil {
missing = append(missing, fmt.Sprintf("%s: the controller's composition %s: %v", m, key, err))
continue
}
comps[key] = comp
}
if len(missing) > 0 {
a.Refused = "returning to the controller's compositions, " + strings.Join(missing, "; ")
a.Verdict = delivery.CandidateRefused
return a
}
cmp := delivery.CompareComposed(comps, running)
a.Returns = delivery.ReturnsToController
a.Same, a.Differs, a.Added, a.Removed = cmp.Same, cmp.Differs, cmp.Added, cmp.Removed
a.Verdict = delivery.VerdictOf(a)
return a
}
// sameValue says whether two sealed values open, with this node's key, to the same value; unknown when either does not
// open (or the node holds no opener). Only the comparison leaves: never a value (issue 1126, the shadow shows its work).
func (c *Composer) sameValue(a, b string) (same, known bool) {
internal/userlist/userlist.go
+58 -220
View File
@@ -1,37 +1,25 @@
// Package userlist is the bridge of novox/hq ADR 0547 §8: the node agent of the node holding mesh-broker applies the
// bus's user list from the newest composition of the bus module the controller writes for its node, alone and with no
// want, so a new account is in force on the bus without anybody's hand.
// Package userlist is the bridge of novox/hq ADR 0547 §8: the node agent of the node holding mesh-broker writes the bus's
// user list alone, with no want, so a new account is in force on the bus without anybody's hand.
//
// **Why it exists.** The list reached the bus only through a want of the bus module, and ADR 0539 §15 bars the assigner
// from writing one: on 2026-10-25 it stood from 11:23 to 20:50 and every new account was refused until the operator put
// it back by hand (issue 1945). The bus module's entrypoint already reloads the server in place when the file changes;
// what was missing is a writer of the file that does not wait on a want.
//
// **It watches, it never polls.** It follows the node's composition bucket. Each composition is verified as the agent
// verifies it (signed by the controller's or the operator's key) before anything of it is read. Of those that carry the
// bus's user list (a file resource `bus-users`), the newest by revision is the source; its list is written alone
// (apply.WriteAlone) when it differs from the file on the machine, and what was written is reported in the bus
// module's have. It looks again after every apply of the bus module, since an apply of an older composition writes an
// older list, and when asked (Again).
// **The grants part's list alone** (ADR 0547 clauses 7, 8 and 11, ADR 0571 clause 8; issue 1126: the controller ended by
// its unassignment and its list is written by nobody). It watches the grants part's one key `user-list.sealed`: the list
// is the object the key names, read by its name, opened with this node's sealing key and checked against the key's
// digest, and written alone (mode 0600, apply.WriteAlone) only when the file's digest differs. A key that cannot be read
// or is deleted is said, the key last heard stands, and the list loaded stays; with no key ever heard nothing is
// written. The list a composition carries is never written.
//
// It reads no want and writes none, moves no fingerprint, and touches nothing of the bus module but its user list.
//
// **It hands the loaded list's hashes to the vault** (ADR 0547 clause 9, issue 1945): once the vault's key is known
// (VaultKey), the have of the bus module carries the bcrypt hash of every user of the list on the machine, sealed to
// that key (delivery.AccountsHanded), so the vault adopts the accounts it holds no value for. It is sealed again only
// when the list on the machine or the vault's key changes.
//
// **From take-over it follows the grants part's list instead** (ADR 0547 clauses 7 and 8, issue 1945). On the node whose
// compositions carry the list, it also watches the grants part's one key `user-list.sealed`. While that key says
// FollowGrants, the list is the object the key names, read by its name, opened with this node's sealing key and checked
// against the key's digest, and written to the same file (mode 0600) only when the file's digest differs; otherwise the
// controller's compositions are the source, as before. Never both. The have says which it follows and this node's key,
// **Where it writes.** The file is the user-list resource (`bus-users`) of the bus module's composition in the node's
// composition bucket, verified as the agent verified it before anything of it is read: what tells the bus node it is
// one, and the file's place. The have of that module says what was written, which list it follows and this node's key,
// which the grants part seals to.
//
// **From the lease write it never follows the controller's list again** (ADR 0571 clause 8, ADR 0547 clause 11). The
// grants part's key then says the lease is handed (delivery.GrantsUserList.LeaseHanded): a key that later cannot be read or
// is deleted is said, the key last heard stands, and the list loaded stays; no list is written from a composition of the
// controller.
// **It watches, it never polls**, and looks again after every apply of the bus module and when asked (Again). It reads no
// want and writes none, moves no fingerprint, and touches nothing of the bus module but its user list.
package userlist
import (
@@ -42,7 +30,6 @@ import (
"encoding/json"
"fmt"
"regexp"
"slices"
"strings"
"sync"
"time"
@@ -64,11 +51,8 @@ type Written = delivery.UserListWritten
type Env struct {
Node string
Composed agent.Watcher
// Trusted is the keys a composition may be signed by, as the agent trusts them.
// Trusted is the keys a composition may be signed by: one any other key signed declares no file.
Trusted func() []ed25519.PublicKey
// Resolve is the list a user-list resource holds: its content, or the object its digest names, fetched and
// verified (the controller sends a file over 64 KiB by digest, ADR 0486 §1, and the live list is over it).
Resolve func(ctx context.Context, resource json.RawMessage) ([]byte, error)
// OnDisk is what the resource's file holds on the machine now; nil when there is none.
OnDisk func(resource json.RawMessage) ([]byte, error)
// Write writes the user list's file resource of a module's composition alone, with body as its content, when it
@@ -77,7 +61,7 @@ type Env struct {
Write func(ctx context.Context, module string, resource json.RawMessage, body []byte) (digest string, changed bool, err error)
// Report says what was written, in the module's have.
Report func(ctx context.Context, module string, w Written)
// Grants watches the grants part's state (its key delivery.GrantsUserListKey); nil follows the controller's alone.
// Grants watches the grants part's state (its key delivery.GrantsUserListKey); nil writes nothing.
Grants agent.Watcher
// Fetch reads an object of the grants part's bucket by its name; Open opens a box sealed to this node's key; Key is
// this node's sealing key's public half (SPKI PEM), said in the have so the grants part seals to it ("" unknown).
@@ -90,11 +74,8 @@ type Env struct {
Kept func() ([]byte, uint64)
Keep func(value []byte, rev uint64)
Backoff func(try int) time.Duration
// VaultKey is the vault's own public key (SPKI PEM) as its answer to node-key gave it, "" while none is known: the
// key the loaded list's hashes are handed sealed to. Nil hands nothing.
VaultKey func() string
Now func() time.Time
Say func(string)
Now func() time.Time
Say func(string)
}
type source struct {
@@ -117,11 +98,9 @@ type Follower struct {
state Written
mod string
said map[string]bool
// grants is the grants part's key as last heard (nil when none), and following the source last taken: FollowGrants
// or FollowController.
// grants is the grants part's key as last heard (nil when none).
grants *delivery.GrantsUserList
grantsRev uint64
following string
// unwatched is why the grants part's key is not watched now; empty while it is, or where it need not be.
unwatched string
}
@@ -174,8 +153,8 @@ func (f *Follower) Run(ctx context.Context) error {
caught := false
// The grants part's key, on the node whose compositions carry the list: the bus node alone is granted it (mesh-sdk
// buscompose.GrantsUserListFollows). Until its watch caught up nothing is written, so a new follower (one per link)
// never writes the controller's list after take-over before it heard the switch; a watch refused or ended is tried
// again on the backoff, and meanwhile the follow last heard (kept on the machine) holds.
// never writes a list older than the key; a watch refused or ended is tried again on the backoff, and meanwhile the
// key last heard (kept on the machine) holds.
var grants <-chan agent.Update
var retry <-chan time.Time
opened, grantsCaught, tries := false, false, 0
@@ -254,34 +233,18 @@ func (f *Follower) busNode() bool {
return f.cur.key != "" || len(f.srcs) > 0
}
// takeGrants keeps the grants part's key as heard; a value it cannot read is said and leaves the controller's list
// followed (no list is followed that the node cannot check), unless the key last heard says the lease is handed: then
// the controller's list is never followed again (novox/hq ADR 0571 clause 8, ADR 0547 clause 11), the key as last heard
// stands, and the list loaded stays.
// takeGrants keeps the grants part's key as heard. A key deleted, or a value it cannot read, is said and changes nothing:
// the key as last heard stands and the list loaded stays (novox/hq ADR 0571 clause 8, ADR 0547 clause 11; issue 1126).
func (f *Follower) takeGrants(u agent.Update) {
f.mu.Lock()
defer f.mu.Unlock()
handed := f.grants != nil && f.grants.LeaseHanded
if u.Deleted {
if handed {
f.sayOnceLocked("the grants part's user-list key is deleted, and the lease is handed, so the list loaded is kept")
return
}
f.grantsRev = u.Revision
f.grants = nil
f.keep(nil, 0)
f.sayOnceLocked("the grants part's user-list key is deleted, so the list loaded is kept")
return
}
g, err := delivery.DecodeGrantsUserList(u.Value)
if err != nil {
if handed {
f.sayOnceLocked("the grants part's user-list key cannot be read, and the lease is handed, so the list loaded is kept: " + err.Error())
return
}
f.grantsRev = u.Revision
f.grants = nil
f.keep(nil, 0)
f.sayOnceLocked("the grants part's user-list key cannot be read, so the controller's list is followed: " + err.Error())
f.sayOnceLocked("the grants part's user-list key cannot be read, so the list loaded is kept: " + err.Error())
return
}
f.grantsRev = u.Revision
@@ -296,21 +259,6 @@ func (f *Follower) keep(value []byte, rev uint64) {
}
}
// Following is which list the follower takes now: the grants part's while its key says so or says the lease is handed, the
// controller's otherwise.
func (f *Follower) Following() string {
f.mu.Lock()
defer f.mu.Unlock()
return f.followingLocked()
}
func (f *Follower) followingLocked() string {
if f.grants != nil && (f.grants.Follow == delivery.FollowGrants || f.grants.LeaseHanded) {
return delivery.FollowGrants
}
return delivery.FollowController
}
func (f *Follower) unwatchedNow() string {
f.mu.Lock()
defer f.mu.Unlock()
@@ -369,103 +317,38 @@ func (f *Follower) take(u agent.Update) {
f.srcs[u.Key] = source{key: u.Key, module: module, rev: u.Revision, resource: raw, wanted: wanted}
}
// settle writes the newest source's list when it is newer than the one written, or the one written again when asked.
// A source never goes back: a key deleted, or a newer one refused, leaves the list written as it is, since an older
// list would take away accounts a newer one gave.
func (f *Follower) settle(ctx context.Context, again bool) {
f.mu.Lock()
follow := f.followingLocked()
if follow != f.following {
// The switch turned: the other source is written, though neither changed.
again = true
}
f.mu.Unlock()
if follow == delivery.FollowGrants {
f.settleGrants(ctx, again)
return
}
f.mu.Lock()
newest := f.cur
for _, s := range f.srcs {
if s.rev > newest.rev {
newest = s
}
}
if newest.key == "" || (newest.rev == f.cur.rev && f.written && !again) {
f.mu.Unlock()
return
}
f.cur = newest
at := fmt.Sprintf("%s@%d", newest.key, newest.rev)
f.mu.Unlock()
w := Written{Source: newest.key, Revision: newest.rev, Wanted: newest.wanted, At: f.env.Now().UTC(),
Follows: delivery.FollowController, Key: f.key(), Unwatched: f.unwatchedNow()}
var changed bool
body, err := f.env.Resolve(ctx, newest.resource)
if err == nil {
w.Users = Users(body)
err = f.guard(newest.resource, body)
if err != nil {
// Not written: what is on the machine stays, and the have says why, loudly.
if disk, derr := f.env.OnDisk(newest.resource); derr == nil && disk != nil {
w.Digest = Digest(disk)
}
}
}
var loaded []byte
if err == nil {
w.Digest, changed, err = f.env.Write(ctx, newest.module, newest.resource, body)
if err == nil {
loaded = body
}
} else if body != nil && f.env.OnDisk != nil {
// Refused: the list on the machine stays, and it is the one the bus has loaded.
loaded, _ = f.env.OnDisk(newest.resource)
}
if err != nil {
w.Error = err.Error()
}
f.mu.Lock()
prev := f.state
f.mu.Unlock()
w.Handed = f.hand(loaded, prev.Handed)
f.mu.Lock()
f.mod, f.state = newest.module, w
f.written = err == nil
f.following = delivery.FollowController
f.mu.Unlock()
if err != nil {
f.sayOnce(fmt.Sprintf("the bus's user list of %s could not be written: %v", at, err))
} else if changed {
f.env.Say(fmt.Sprintf("the bus's user list from %s is written alone (%d users, %s); the bus reloads it itself",
at, w.Users, short(w.Digest)))
}
// Reported when what it says changed: a look again that found the file as it was says nothing new.
if f.env.Report != nil && (changed || reportable(prev, w)) {
f.env.Report(ctx, newest.module, w)
}
}
// settle writes the grants part's list: when its key moved, or when asked.
func (f *Follower) settle(ctx context.Context, again bool) { f.settleGrants(ctx, again) }
// reportable says a have's user list says something new: a look again that found the file as it was says nothing.
func reportable(prev, w Written) bool {
return prev.Source != w.Source || prev.Revision != w.Revision || prev.Digest != w.Digest || prev.Error != w.Error ||
prev.Follows != w.Follows || prev.Key != w.Key || handedKey(prev.Handed) != handedKey(w.Handed) ||
prev.Unwatched != w.Unwatched
prev.Follows != w.Follows || prev.Key != w.Key || prev.Unwatched != w.Unwatched
}
// settleGrants writes the grants part's list, the object its key names, into the file the controller's composition of
// the bus module declares, made 0600 (ADR 0547 clause 7): when the key moved, when the switch turned, or when asked.
// settleGrants writes the grants part's list, the object its key names, into the file the bus module's composition
// declares, made 0600 (ADR 0547 clause 7): when the key moved, or when asked. With no key heard it writes nothing and
// says why in the have.
func (f *Follower) settleGrants(ctx context.Context, again bool) {
f.mu.Lock()
g, rev := *f.grants, f.grantsRev
var g delivery.GrantsUserList
heard := f.grants != nil
if heard {
g = *f.grants
}
rev := f.grantsRev
file := f.cur
for _, s := range f.srcs {
if s.rev > file.rev {
file = s
}
}
if !again && f.following == delivery.FollowGrants && f.state.Revision == rev && f.written {
if file.key == "" {
f.mu.Unlock()
return // no composition of this node declares the file: not the bus node
}
f.cur = file
if !again && f.state.Source != "" && f.state.Revision == rev && f.written {
f.mu.Unlock()
return
}
@@ -477,8 +360,10 @@ func (f *Follower) settleGrants(ctx context.Context, again bool) {
var changed bool
resource, rerr := mode0600(file.resource)
switch {
case file.key == "" || rerr != nil:
err = fmt.Errorf("no composition of this node declares the bus's user-list file, so the grants part's list has nowhere to be written")
case rerr != nil:
err = fmt.Errorf("the bus's user-list file this node's composition declares cannot be read: %w", rerr)
case !heard:
err = fmt.Errorf("the grants part's user-list key is not heard yet; the list on the machine is kept")
case g.Digest == "":
err = fmt.Errorf("the grants part's key names no list yet; the list on the machine is kept")
case f.env.Fetch == nil || f.env.Open == nil:
@@ -506,7 +391,7 @@ func (f *Follower) settleGrants(ctx context.Context, again bool) {
}
if err == nil {
w.Digest, changed, err = f.env.Write(ctx, file.module, resource, body)
} else if w.Digest == "" && file.key != "" && f.env.OnDisk != nil {
} else if w.Digest == "" && f.env.OnDisk != nil {
if disk, derr := f.env.OnDisk(resource); derr == nil && disk != nil {
w.Digest = Digest(disk)
}
@@ -516,11 +401,10 @@ func (f *Follower) settleGrants(ctx context.Context, again bool) {
}
f.mu.Lock()
prev := f.state
// The grants part's list is not handed to the vault: the handing is of what the controller's list loaded (clause 9).
// The grants part's list is not handed to the vault: a handing written before stands (clause 9).
w.Handed = prev.Handed
f.mod, f.state = file.module, w
f.written = err == nil
f.following = delivery.FollowGrants
f.mu.Unlock()
at := fmt.Sprintf("%s@%d", delivery.GrantsUserListSource, rev)
if err != nil {
@@ -529,21 +413,17 @@ func (f *Follower) settleGrants(ctx context.Context, again bool) {
f.env.Say(fmt.Sprintf("the bus's user list from the grants part (%s, revision %d) is written alone (%d users, %s); the bus reloads it itself",
at, g.Revision, w.Users, short(w.Digest)))
}
if f.env.Report != nil && file.module != "" && (changed || reportable(prev, w)) {
if f.env.Report != nil && (changed || reportable(prev, w)) {
f.env.Report(ctx, file.module, w)
}
}
// Hold is c as the apply may write it (C3 of the switch's review): while the grants part's list is followed, the bus
// module's composition from the controller carries the controller's list, so its user-list resource holds the list on the
// machine instead (made 0600), or is left out when the machine holds none. A list on the machine that cannot be read
// is an error: left out, the apply would remove the file as an orphan and the bus would refuse every connection at its
// next reload, so the bus module's apply waits instead. Any other composition, or the controller's list followed, is c
// unchanged.
// Hold is c as the apply may write it (C3 of the switch's review): the list a composition carries is never written, so
// its user-list resource holds the list on the machine instead (made 0600), or is left out when the machine holds none.
// A list on the machine that cannot be read is an error: left out, the apply would remove the file as an orphan and the
// bus would refuse every connection at its next reload, so the bus module's apply waits instead. A composition without
// a user list is c unchanged.
func (f *Follower) Hold(c delivery.Composed) (delivery.Composed, error) {
if f.Following() != delivery.FollowGrants {
return c, nil
}
out := c
out.Resources = nil
for _, raw := range c.Resources {
@@ -585,54 +465,12 @@ func mode0600(raw json.RawMessage) (json.RawMessage, error) {
return json.Marshal(r)
}
// hand is the handing of the list on the machine to the vault: the one before when it is of the same list and key, a
// new one sealed now otherwise, and none while the vault's key is not known or the list holds no hash.
func (f *Follower) hand(loaded []byte, before *delivery.AccountsHanded) *delivery.AccountsHanded {
if f.env.VaultKey == nil || loaded == nil {
return before
}
key := f.env.VaultKey()
if key == "" {
return before
}
from := Digest(loaded)
if before != nil && before.From == from && before.To == delivery.KeyFingerprint(key) {
return before
}
accounts := delivery.HandedAccountsOf(loaded)
if len(accounts) == 0 {
return nil
}
h, err := delivery.HandAccounts(accounts, key, from, f.env.Now())
if err != nil {
f.sayOnce(fmt.Sprintf("the loaded list's hashes are not handed to the vault: %v", err))
return before
}
f.env.Say(fmt.Sprintf("the hashes of the %d users of the bus's loaded list (%s) are handed to the vault, sealed to "+
"its key %s, in the bus module's have", h.Users, short(from), short(h.To)))
return &h
}
// handedKey is what a handing says that a report compares: its list and key.
func handedKey(h *delivery.AccountsHanded) string {
if h == nil {
return ""
}
return h.From + " " + h.To
}
// ControllerUser is the controller's own user in the list: a list without it would lock the controller out of the bus.
const ControllerUser = "controller"
// guard refuses a list that would lock the mesh out of its own bus: one without the controller's own user, or one that
// drops more than half the users the file on the machine holds. A list that shrinks that far is a composer that read
// half the mesh, not half the mesh gone; it waits for a person, and the bus keeps the list it has.
// guard refuses a list that would lock the mesh out of its own bus: one that drops more than half the users the file on
// the machine holds. A list that shrinks that far is a composer that read half the mesh, not half the mesh gone; it
// waits for a person, and the bus keeps the list it has. (A list without the controller's user is no longer refused:
// the controller ended, issue 1126.)
func (f *Follower) guard(resource json.RawMessage, body []byte) error {
names := Names(body)
if !slices.Contains(names, ControllerUser) {
return fmt.Errorf("refused: the list holds no user %q, and written it would lock the controller out of the "+
"bus; the list on the machine is kept", ControllerUser)
}
if f.env.OnDisk == nil {
return nil
}
internal/userlist/userlist_test.go
+36 -128
View File
@@ -59,19 +59,6 @@ func (f *feed) Watch(context.Context, string) (<-chan agent.Update, error) {
return f.ch, nil
}
// hashed is a list whose users carry a hash of bcrypt's form, as the composer writes them.
func hashed(users ...string) string {
var b strings.Builder
b.WriteString("authorization {\n users = [\n")
for _, u := range users {
b.WriteString(` { user: "` + u + `", password: "$2a$11$` + strings.Repeat("h", 53) + `", permissions: {` + "\n")
}
b.WriteString(" ]\n}\n")
return b.String()
}
var vault, _ = seal.GenerateKeyPair()
func list(users ...string) string {
var b strings.Builder
b.WriteString("authorization {\n users = [\n")
@@ -132,9 +119,8 @@ type gput struct {
otherKey, badDigest bool
// after puts it once the grants part's list was written, as a hand-back comes after a take-over.
after bool
// handed says the controller's lease is handed (ADR 0582 clause 3); unreadable and deleted make the key a value that
// does not read, or a deleted key, instead of a list (774/S31).
handed, unreadable, deleted bool
// unreadable and deleted make the key a value that does not read, or a deleted key, instead of a list (774/S31).
unreadable, deleted bool
}
type put struct {
@@ -145,13 +131,12 @@ type put struct {
}
// Defends novox/hq ADR 0547 §8: the bus node writes the newest signed list alone, only when it changed, and reports it.
func TestTheBusNodeWritesTheNewestSignedUserListAlone(t *testing.T) {
func TestTheBusNodeWritesTheGrantsPartsUserListAlone(t *testing.T) {
one, two, three := list("controller", "a"), list("controller", "a", "b"), list("controller", "a", "b", "c")
five := list("controller", "a", "b", "c", "d")
for _, c := range []struct {
name string
before, after []put
again bool
failWrite bool
onDisk string
wantWrites []string // contents written, in order
@@ -166,83 +151,48 @@ func TestTheBusNodeWritesTheNewestSignedUserListAlone(t *testing.T) {
wantUnwatched bool
// wantSource and wantFollows are what the last have says it took the list from and follows.
wantSource, wantFollows string
// vault gives the follower the vault's key; wantHanded is how many users it then hands from the list on the
// machine (ADR 0547 clause 9).
vault bool
wantHanded int
}{
{name: "the list there is written once", before: []put{{module: "nats", fp: fpOf("a"), content: two}},
wantWrites: []string{two}, wantUsers: 3},
{name: "a newer revision is written, the older fingerprint is not",
before: []put{{module: "nats", fp: fpOf("a"), content: one}, {module: "nats", fp: fpOf("b"), content: two}},
after: []put{{module: "nats", fp: fpOf("a"), content: three}},
wantWrites: []string{two, three}, wantUsers: 4},
{name: "a composition signed by a stranger is no source",
before: []put{{module: "nats", fp: fpOf("a"), content: one}},
after: []put{{module: "nats", fp: fpOf("b"), content: three, stranger: true}},
wantWrites: []string{one}, wantUsers: 2},
{name: "a composition without the list is ignored", before: []put{{module: "web", fp: fpOf("a"), content: ""}}},
{name: "a deleted key is no source",
before: []put{{module: "nats", fp: fpOf("a"), content: one}, {module: "nats", fp: fpOf("b"), content: two}},
after: []put{{module: "nats", fp: fpOf("b"), content: "", deleted: true}, {module: "nats", fp: fpOf("a"), content: three}},
wantWrites: []string{two, three}, wantUsers: 4},
{name: "asked again, the same source is written again", before: []put{{module: "nats", fp: fpOf("a"), content: two}},
again: true, wantWrites: []string{two, two}, wantUsers: 3},
{name: "a list sent by digest counts the users of what it names",
before: []put{{module: "nats", fp: fpOf("a"), content: three, byDigest: true}},
wantWrites: []string{three}, wantUsers: 4},
{name: "another node's composition under this node's key is no source",
before: []put{{module: "nats", fp: fpOf("a"), content: two, asNode: "ace"}}},
{name: "another module's composition under the bus module's key is no source",
before: []put{{module: "nats", fp: fpOf("a"), content: two, asModule: "web"}}},
{name: "a list without the controller is refused and the file kept",
before: []put{{module: "nats", fp: fpOf("a"), content: list("a", "b")}},
onDisk: two, wantUsers: 2, wantErr: "holds no user \"controller\"", wantRefused: true},
{name: "a list that drops more than half is refused and the file kept",
before: []put{{module: "nats", fp: fpOf("a"), content: one}},
onDisk: five, wantUsers: 2, wantErr: "drops more than half", wantRefused: true},
{name: "a list that drops half is written", before: []put{{module: "nats", fp: fpOf("a"), content: two}},
onDisk: list("controller", "a", "b", "c", "d", "e"), wantWrites: []string{two}, wantUsers: 3},
{name: "the loaded list's hashes are handed sealed to the vault's key",
before: []put{{module: "nats", fp: fpOf("a"), content: hashed("controller", "node.ace")}}, vault: true,
wantWrites: []string{hashed("controller", "node.ace")}, wantUsers: 2, wantHanded: 2},
{name: "a refused list hands the list on the machine",
before: []put{{module: "nats", fp: fpOf("a"), content: list("a", "b")}}, vault: true,
onDisk: hashed("controller", "a", "b"), wantUsers: 2, wantErr: "holds no user", wantRefused: true, wantHanded: 3},
{name: "a failed write is said and kept", before: []put{{module: "nats", fp: fpOf("a"), content: two}}, failWrite: true,
wantWrites: []string{two}, wantUsers: 3, wantErr: "disk full"},
// ADR 0547 clauses 7, 8: never both lists; the grants part's from take-over, the controller's otherwise.
{name: "from take-over the grants part's list is written into the same file",
// Issue 1126, A6 (ADR 0547 clauses 7, 8, 11): the grants part's list alone, written into the file the bus module's
// composition declares; the list a composition carries is never written, whatever becomes of the grants key.
{name: "the grants part's list is written into the file a composition declares",
before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grants: []gput{{follow: delivery.FollowGrants, content: three}},
wantWrites: []string{three}, wantUsers: 4, wantSource: delivery.GrantsUserListSource, wantFollows: delivery.FollowGrants},
{name: "a hand-back writes the newest controller composition once",
{name: "a key that says follow the controller still writes the grants part's list",
before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grants: []gput{{follow: delivery.FollowController, content: three}},
wantWrites: []string{three}, wantUsers: 4, wantSource: delivery.GrantsUserListSource, wantFollows: delivery.FollowGrants},
{name: "a newer composition's list is not written",
before: []put{{module: "nats", fp: fpOf("a"), content: two}},
grants: []gput{{follow: delivery.FollowGrants, content: three}, {follow: delivery.FollowController, content: three, after: true}},
wantWrites: []string{three, two}, wantUsers: 3, wantSource: "novox.nats." + fpOf("a")[len(delivery.SourcePrefix2):], wantFollows: delivery.FollowController},
{name: "a refused watch with no key kept follows the controller's list and says it is not watched",
after: []put{{module: "nats", fp: fpOf("b"), content: five}},
grants: []gput{{follow: delivery.FollowGrants, content: three}},
wantWrites: []string{three}, wantUsers: 4, wantSource: delivery.GrantsUserListSource, wantFollows: delivery.FollowGrants},
{name: "a composition without the list is ignored", before: []put{{module: "web", fp: fpOf("a"), content: ""}},
grants: []gput{{follow: delivery.FollowGrants, content: three}}},
{name: "a composition signed by a stranger declares no file",
before: []put{{module: "nats", fp: fpOf("a"), content: two, stranger: true}}, grants: []gput{{follow: delivery.FollowGrants, content: three}}},
{name: "an unreadable grants key with none heard before writes nothing, never the composition's list",
before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grants: []gput{{unreadable: true}},
onDisk: two, wantErr: "key is not heard", wantRefused: true, wantSource: delivery.GrantsUserListSource},
{name: "a refused watch with no key kept writes nothing and says it is not watched",
before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grantsRefused: true, wantUnwatched: true,
wantWrites: []string{two}, wantUsers: 3, wantSource: "novox.nats." + fpOf("a")[len(delivery.SourcePrefix2):], wantFollows: delivery.FollowController},
{name: "a refused watch keeps the follow last heard, never the controller's list",
onDisk: two, wantErr: "key is not heard", wantRefused: true, wantSource: delivery.GrantsUserListSource},
{name: "a refused watch keeps the key last heard",
before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grantsRefused: true, wantUnwatched: true,
kept: &gput{follow: delivery.FollowGrants, content: three},
wantWrites: []string{three}, wantUsers: 4, wantSource: delivery.GrantsUserListSource, wantFollows: delivery.FollowGrants},
{name: "in shadow the grants part's list is not taken",
before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grants: []gput{{follow: delivery.FollowController, content: three}},
wantWrites: []string{two}, wantUsers: 3, wantSource: "novox.nats." + fpOf("a")[len(delivery.SourcePrefix2):], wantFollows: delivery.FollowController},
// 774/S31 (ADR 0571 clause 8, ADR 0547 clause 11): once the key says the lease is handed, the controller's list is never
// followed again, whatever becomes of the key; without the flag the controller's list is still the fallback.
{name: "a handed lease: a grants key that turns unreadable keeps the loaded list and never writes the controller's",
before: []put{{module: "nats", fp: fpOf("a"), content: two}},
grants: []gput{{follow: delivery.FollowGrants, content: three, handed: true}, {unreadable: true, after: true}},
wantWrites: []string{three}, wantUsers: 4, wantSource: delivery.GrantsUserListSource, wantFollows: delivery.FollowGrants},
{name: "a handed lease: a grants key that is deleted keeps the loaded list and never writes the controller's",
before: []put{{module: "nats", fp: fpOf("a"), content: two}},
grants: []gput{{follow: delivery.FollowGrants, content: three, handed: true}, {deleted: true, after: true}},
wantWrites: []string{three}, wantUsers: 4, wantSource: delivery.GrantsUserListSource, wantFollows: delivery.FollowGrants},
{name: "no handed lease: a grants key that turns unreadable falls back to the controller's list",
{name: "a grants key that turns unreadable keeps the loaded list",
before: []put{{module: "nats", fp: fpOf("a"), content: two}},
grants: []gput{{follow: delivery.FollowGrants, content: three}, {unreadable: true, after: true}},
wantWrites: []string{three, two}, wantUsers: 3, wantSource: "novox.nats." + fpOf("a")[len(delivery.SourcePrefix2):], wantFollows: delivery.FollowController},
wantWrites: []string{three}, wantUsers: 4, wantSource: delivery.GrantsUserListSource, wantFollows: delivery.FollowGrants},
{name: "a grants key that is deleted keeps the loaded list",
before: []put{{module: "nats", fp: fpOf("a"), content: two}},
grants: []gput{{follow: delivery.FollowGrants, content: three}, {deleted: true, after: true}},
wantWrites: []string{three}, wantUsers: 4, wantSource: delivery.GrantsUserListSource, wantFollows: delivery.FollowGrants},
{name: "a grants list that drops more than half is refused and the file kept",
before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grants: []gput{{follow: delivery.FollowGrants, content: one}},
onDisk: five, wantUsers: 2, wantErr: "drops more than half", wantRefused: true, wantSource: delivery.GrantsUserListSource},
{name: "a failed write is said and kept", before: []put{{module: "nats", fp: fpOf("a"), content: two}},
grants: []gput{{follow: delivery.FollowGrants, content: three}}, failWrite: true,
wantWrites: []string{three}, wantUsers: 4, wantErr: "disk full", wantSource: delivery.GrantsUserListSource},
{name: "a grants list that does not open to its digest is refused and the file kept",
before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grants: []gput{{follow: delivery.FollowGrants, content: three, badDigest: true}},
onDisk: two, wantErr: "could not be had", wantRefused: true, wantSource: delivery.GrantsUserListSource},
@@ -293,7 +243,7 @@ func TestTheBusNodeWritesTheNewestSignedUserListAlone(t *testing.T) {
}
sealed[delivery.UserListObjectName(digest)] = object
raw, err := delivery.EncodeGrantsUserList(delivery.GrantsUserList{Follow: g.follow, Digest: digest,
Object: delivery.UserListObjectName(digest), SealedTo: delivery.SealingKeyID(to), Node: node, Revision: uint64(i + 1), LeaseHanded: g.handed})
Object: delivery.UserListObjectName(digest), SealedTo: delivery.SealingKeyID(to), Node: node, Revision: uint64(i + 1)})
if err != nil {
t.Fatal(err)
}
@@ -323,18 +273,6 @@ func TestTheBusNodeWritesTheNewestSignedUserListAlone(t *testing.T) {
},
Key: func() string { return nodeKey.PublicKey },
Trusted: func() []ed25519.PublicKey { return []ed25519.PublicKey{controllerPub} },
Resolve: func(_ context.Context, r json.RawMessage) ([]byte, error) {
var res struct{ Content, Digest string }
_ = json.Unmarshal(r, &res)
if res.Digest != "" {
body, ok := objects[res.Digest]
if !ok {
return nil, errors.New("no object " + res.Digest)
}
return []byte(body), nil
}
return []byte(res.Content), nil
},
OnDisk: func(json.RawMessage) ([]byte, error) {
if c.onDisk == "" {
return nil, nil
@@ -357,12 +295,6 @@ func TestTheBusNodeWritesTheNewestSignedUserListAlone(t *testing.T) {
reported <- struct{}{}
},
Now: func() time.Time { return time.Date(2026, 10, 25, 21, 50, 0, 0, time.UTC) },
VaultKey: func() string {
if c.vault {
return vault.PublicKey
}
return ""
},
})
update := func(p put) agent.Update {
f.rev++
@@ -411,14 +343,6 @@ func TestTheBusNodeWritesTheNewestSignedUserListAlone(t *testing.T) {
gf.ch <- u
}
}
if c.again {
// After the first write is reported, so the look again is not folded into it.
select {
case <-reported:
case <-time.After(5 * time.Second):
}
fl.Again()
}
deadline := time.After(5 * time.Second)
for {
mu.Lock()
@@ -489,22 +413,6 @@ func TestTheBusNodeWritesTheNewestSignedUserListAlone(t *testing.T) {
(c.wantFollows != "" && st.Follows != c.wantFollows)) {
t.Fatalf("the have says source %q follows %q key %t; want %q %q", st.Source, st.Follows, st.Key == nodeKey.PublicKey, c.wantSource, c.wantFollows)
}
switch {
case c.wantHanded == 0 && st.Handed != nil:
t.Fatalf("handed with no vault key or no hash: %+v", st.Handed)
case c.wantHanded > 0:
loaded := c.onDisk
if !c.wantRefused {
loaded = c.wantWrites[len(c.wantWrites)-1]
}
if st.Handed == nil || st.Handed.From != Digest([]byte(loaded)) {
t.Fatalf("handed %+v, want from the list on the machine", st.Handed)
}
opened, err := delivery.OpenHanded(*st.Handed, vault.PublicKey, vault.PrivateKey)
if err != nil || len(opened) != c.wantHanded || opened[0].User != "controller" {
t.Fatalf("opened %+v, %v", opened, err)
}
}
if c.wantRefused {
if !strings.Contains(st.Error, c.wantErr) || st.Digest != Digest([]byte(c.onDisk)) || st.Users != c.wantUsers {
t.Fatalf("refused list's state %+v", st)