L3: the node trusts the vault's key, serves the node seat node-engine and keeps its acts on the bus (issue 774, ADR 0561), rebased onto L2 #168

Merged
mesh-tools merged 14 commits from feat/774-vault-mints-l2 into main 2026-10-29 07:44:19 +00:00
10 changed files with 623 additions and 21 deletions
Showing only changes of commit 479ac71050 - Show all commits
+111 -2
View File
@@ -1,16 +1,27 @@
package main
import (
"context"
"crypto/ed25519"
"crypto/x509"
"encoding/base64"
"encoding/json"
"encoding/pem"
"os"
"path/filepath"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/overlay"
"github.com/novox/mesh-host/internal/identity"
)
// Issue 1126 (ADR 0555 §3): the mesh's own certificate a composition carries is kept for the machine's record, and the
// record's keys read it; a composition carrying none changes nothing.
// record's keys read it; a composition carrying none changes nothing. And (L3 s6, s7; ADR 0550 §2) the record says what
// the node trusts a mesh signature from, and the node keeps the certificate the vault signed for it.
func TestTheAppliedAuthorityIsKeptForTheRecord(t *testing.T) {
state := filepath.Join(t.TempDir(), "state", "state.json")
res := func(id, content string) json.RawMessage {
@@ -26,7 +37,105 @@ func TestTheAppliedAuthorityIsKeptForTheRecord(t *testing.T) {
if err := keepAuthority(state, delivery.Composed{Resources: []json.RawMessage{res("ca-trust.certificate-authority", "CA-1")}}); err != nil {
t.Fatal(err)
}
if got := recordKeys(state)(); got.Authority != "CA-1" {
if got := recordKeys(state, nil)(); got.Authority != "CA-1" {
t.Errorf("the record's keys read the authority %q", got.Authority)
}
// L3 s6: before the vault's signers are read the node trusts the controller's key alone; once read, a message the
// vault signed is accepted and the record names both; once the lease is handed away the controller's key is refused
// and the record names the vault's alone. Each change tells the record's writer.
controllerPub, controller, _ := ed25519.GenerateKey(nil)
vaultPub, vault, _ := ed25519.GenerateKey(nil)
trust := newMeshTrust(controllerPub)
told := 0
trust.onChange(func() { told++ })
keys := recordKeys(state, trust.Trusts)
ctrlFP, vaultFP := delivery.SignerFingerprint(controllerPub), delivery.SignerFingerprint(vaultPub)
if got := keys().Trusts; !delivery.TrustsKey(got, delivery.SignerController, ctrlFP) || delivery.TrustsKey(got, delivery.SignerVault, vaultFP) {
t.Errorf("before the signers are read the record trusts %v", got)
}
enrol, _ := delivery.SignContext(delivery.KindEnrolment)
msg := []byte("a reply")
if _, err := trust.Verify(delivery.KindEnrolment, msg, delivery.SignMessage(vault, enrol, msg)); err == nil {
t.Error("a vault-signed message is accepted before the vault's key is read")
}
vaultCA := testAuthority(t, vaultPub, vault)
keyValue, _ := json.Marshal(delivery.NewSignerKey(vaultPub, time.Now()))
fp, _ := delivery.AuthorityFingerprint(vaultCA)
authValue, _ := json.Marshal(delivery.SignerAuthority{Certificate: vaultCA, Fingerprint: fp, Made: time.Now()})
trust.heardSigner(delivery.SignersKeyKey, keyValue, false, func(string) {})
trust.heardSigner(delivery.SignersAuthorityKey, authValue, false, func(string) {})
if s, err := trust.Verify(delivery.KindEnrolment, msg, delivery.SignMessage(vault, enrol, msg)); err != nil || s.Name != delivery.SignerVault {
t.Errorf("a vault-signed message on a node that read signers: %+v, %v", s, err)
}
if got := keys().Trusts; !delivery.TrustsKey(got, delivery.SignerController, ctrlFP) || !delivery.TrustsKey(got, delivery.SignerVault, vaultFP) ||
trust.VaultAuthority() != vaultCA || told != 2 {
t.Errorf("after the signers are read the record trusts %v (told %d)", got, told)
}
handed, _ := json.Marshal(buscompose.LeaseHolder{HandedTo: "mesh-vault", HandedBy: "person.operator"})
trust.heardLease(handed, true, nil)
if _, err := trust.Verify(delivery.KindEnrolment, msg, delivery.SignMessage(controller, enrol, msg)); err == nil || trust.Controller() != nil {
t.Error("after the lease is handed away the controller's key is still trusted")
}
if got := keys().Trusts; delivery.TrustsKey(got, delivery.SignerController, ctrlFP) || !delivery.TrustsKey(got, delivery.SignerVault, vaultFP) || told != 3 {
t.Errorf("after the lease is handed away the record trusts %v (told %d)", got, told)
}
trust.heardLease(handed, true, nil)
if told != 3 {
t.Error("a lease heard again unchanged wrote the record again")
}
// A signers' key naming another fingerprint is not trusted.
bad, _ := json.Marshal(delivery.SignerKey{Key: base64.StdEncoding.EncodeToString(controllerPub), Fingerprint: vaultFP})
trust.heardSigner(delivery.SignersKeyKey, bad, false, func(string) {})
if !delivery.TrustsKey(keys().Trusts, delivery.SignerVault, vaultFP) {
t.Error("a signers' key that does not check replaced the vault's")
}
// L3 s7: the certificate the vault signs for this node's internal name and serving key is kept where the record reads
// it, and the record is written again; one signed by another authority is refused and nothing is kept.
serving, err := identity.GenerateServingKey()
if err != nil {
t.Fatal(err)
}
if err := identity.WriteServingKey(identity.ServingKeyPath(state), serving); err != nil {
t.Fatal(err)
}
servingPub, _ := base64.StdEncoding.DecodeString(serving.Public)
signedBy := func(ca string, key ed25519.PrivateKey) string {
parent, _ := delivery.ParseCertificate(ca)
der, err := x509.CreateCertificate(nil, delivery.NodeCertificateTemplate(overlay.InternalName("n1"), servingPub, time.Now().Add(-time.Hour)),
parent, ed25519.PublicKey(servingPub), key)
if err != nil {
t.Fatal(err)
}
return string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}))
}
var asked []any
answer := signedBy(vaultCA, vault)
ask := func(_ context.Context, verb string, args any) (json.RawMessage, error) {
asked = append(asked, verb, args)
return json.Marshal(delivery.SignAnswer{Kind: delivery.KindNodeCertificate, Node: "n1", Certificate: answer})
}
written := 0
if err := certifyThisNode(context.Background(), state, "n1", trust, ask, func() { written++ }); err != nil {
t.Fatal(err)
}
if got := recordKeys(state, nil)(); got.Certificate != answer || written != 1 || len(asked) != 2 || asked[0] != delivery.SignVerb {
t.Errorf("the vault's certificate: kept %v, written %d, asked %v", got.Certificate == answer, written, asked)
}
answer = signedBy(testAuthority(t, controllerPub, controller), controller)
if err := certifyThisNode(context.Background(), state, "n1", trust, ask, func() { written++ }); err == nil || written != 1 {
t.Errorf("a certificate of another authority was kept: %v", err)
}
}
// testAuthority is a self-signed authority certificate for key, as the vault makes its own.
func testAuthority(t *testing.T, pub ed25519.PublicKey, key ed25519.PrivateKey) string {
t.Helper()
tpl := delivery.AuthorityTemplate(pub, time.Now().Add(-time.Hour))
der, err := x509.CreateCertificate(nil, tpl, tpl, pub, key)
if err != nil {
t.Fatal(err)
}
return string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}))
}
+12 -3
View File
@@ -1105,10 +1105,13 @@ func runLink(ctx context.Context, opts options) error {
// the bus, answer what the controller asks of this machine, and say its facts: when a link opens, and when the
// controller asks.
proof := &proving{statePath: opts.state, version: runningVersion(), say: say}
// What this engine trusts a mesh signature from: the controller's key until its lease is handed away, and the vault's
// once read from its signers (novox/hq ADR 0550 §2).
trust := newMeshTrust(mine.Membership.Signer)
queue := &link.Session{
Membership: membership,
// And the public halves and certificate its record says (novox/hq ADR 0555 §3), read at each facts report.
Keys: recordKeys(opts.state),
Keys: recordKeys(opts.state, trust.Trusts),
// And the first have this build writes is what proves it to the launcher (ADR 0539 §9, ADR 0227 §8).
HaveWritten: proof.haveWritten,
// And the operator's hand-over of a directory used as found, carried by the controller from the
@@ -1173,7 +1176,10 @@ func runLink(ctx context.Context, opts options) error {
}
// The node follows its wants: one module at a time from the composition its want names (novox/hq ADR 0539), and the
// busy containers are asked for again soon (ADR 0360 §11).
go nodeAgent(aside, opts, mine.Membership.Signer, queue, sched, say, afterApply, current, factsOfNow)
// And what it trusts, followed on the bus, said in its record whenever it changes (ADR 0550 §2).
trust.onChange(func() { factsOfNow(aside, current.get()) })
go followTrust(aside, queue, trust, say)
go nodeAgent(aside, opts, mine.Membership.Signer, trust, queue, sched, say, afterApply, current, factsOfNow)
go busyPasses(aside, func() {
if ag := current.get(); ag != nil {
ag.Retry(busyModules()...)
@@ -2167,9 +2173,12 @@ func withMissing(st liveness.Statement) liveness.Statement {
// recordKeys reads the public halves of the node's keys and the certificate the mesh issued for its internal name from
// its state directory (novox/hq ADR 0555 §3): what the node's record says of them. A key that cannot be read is left
// empty, never invented; the certificate is there only after the first declaration that carried it.
func recordKeys(state string) func() link.RecordKeys {
func recordKeys(state string, trusts func() []string) func() link.RecordKeys {
return func() link.RecordKeys {
var k link.RecordKeys
if trusts != nil {
k.Trusts = trusts()
}
if id, err := identity.Load(identity.Path(state)); err == nil {
k.TunnelKey = id.Overlay.Public
}
+44 -12
View File
@@ -5,12 +5,14 @@ import (
"crypto/ed25519"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"sync/atomic"
"time"
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
@@ -343,13 +345,14 @@ func resourcesOf(cs []delivery.Composed) []declaration.Resource {
// controller's key the node was enrolled with (novox/hq ADR 0539 §3, §11); a node adopted later has it at its adoption.
const operatorKeysFile = "operator-keys"
// trustedKeys are the keys a composition may be signed by: the controller's, which the node was enrolled with, and the
// operator's, read from operatorKeysFile each time it is asked, so a key that arrives while the agent runs is trusted at
// the next composition. A file that cannot be read trusts the controller's alone, and says so.
func trustedKeys(opts options, controller ed25519.PublicKey) func() []ed25519.PublicKey {
// trustedKeys are the keys a composition may be signed by: the controller's, which the node was enrolled with, while the
// controller holds its lease (ADR 0550 §2: trusted no more once it is handed away), and the operator's, read from
// operatorKeysFile each time it is asked, so a key that arrives while the agent runs is trusted at the next composition.
// A file that cannot be read trusts the controller's alone, and says so.
func trustedKeys(opts options, controllerNow func() ed25519.PublicKey) func() []ed25519.PublicKey {
return func() []ed25519.PublicKey {
keys := []ed25519.PublicKey{}
if len(controller) == ed25519.PublicKeySize {
if controller := controllerNow(); len(controller) == ed25519.PublicKeySize {
keys = append(keys, controller)
}
raw, err := os.ReadFile(filepath.Join(filepath.Dir(opts.state), operatorKeysFile))
@@ -373,13 +376,16 @@ func trustedKeys(opts options, controller ed25519.PublicKey) func() []ed25519.Pu
}
// keptOnce reads the kept declaration once and splits it by module: its resources (the node's own are those that say
// none, ADR 0539 §9) and its filled paths.
// none, ADR 0539 §9) and its filled paths. Once the read is retired (keptRetired) nothing is read.
func keptOnce(opts options, signer ed25519.PublicKey) func() (map[string][]json.RawMessage, map[string][]string) {
var once sync.Once
var by map[string][]json.RawMessage
var filled map[string][]string
return func() (map[string][]json.RawMessage, map[string][]string) {
once.Do(func() {
if keptRetired.Load() {
return
}
raw, err := store.LoadDeclared(store.DeclaredPath(opts.state), signer)
if err != nil {
return
@@ -429,7 +435,7 @@ func splitKept(raw []byte) (map[string][]json.RawMessage, map[string][]string) {
// nodeAgent runs the node agent under the session's links for as long as ctx lasts: it waits for a link, follows the
// node's wants and compositions on it, and begins again with the next link. A watch it cannot make (a bucket this node's
// account may not read) is said, and tried again with a growing wait, as the link is.
func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, session *link.Session, sched *apply.Scheduler,
func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trust *meshTrust, session *link.Session, sched *apply.Scheduler,
say link.Announce, afterApply func(), current *agentNow, sayFacts func(ctx context.Context, ag *agent.Agent)) {
wait := 2 * time.Second
for ctx.Err() == nil {
@@ -446,6 +452,11 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
return
}
var ag *agent.Agent
// The kept release-5 declaration is no longer read once every module the catalogue places here has a have of its
// own (ADR 0550 §2: its signature is the controller's, trusted no more after the lease write).
if retired, why := placedAllHave(linked, session, node); retired && !keptRetired.Swap(true) {
say("the kept declaration is no longer read: " + why)
}
kept := keptOnce(opts, signer)
// The mesh's own certificate the declaration kept from before carries, kept for the record before this machine
// composes its own (ADR 0555 §3): without it a module that asks for a certificate would be stood in for.
@@ -458,7 +469,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
}
// The bus's user list, written alone from the newest composition of the bus module (ADR 0547 §8): on every
// node, and a node whose compositions carry no list writes nothing.
users := userListFollower(opts, session, composedBucket, trustedKeys(opts, signer), say, func() {
users := userListFollower(opts, session, composedBucket, trustedKeys(opts, trust.Controller), say, func() {
// This engine wrote the list, so the bus reloads its accounts and answers no JetStream request for a few
// seconds: what failed in them is tried again once that is over, not at its next wait.
time.AfterFunc(reloadSettles, func() {
@@ -495,7 +506,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
}
return out, nil
},
Trusted: trustedKeys(opts, signer),
Trusted: trustedKeys(opts, trust.Controller),
Backup: backupNode(session),
// What the filling refused before this apply may be named now: the provisioner's client is told to fill
// again (ace, 2026-10-25 21:17).
@@ -572,11 +583,17 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
now.NotComposedEach(run, why, held) // a module held alone marks that module, never the others (issue 1126)
}
},
// And the vault's authority beside the controller's, and this node's certificate asked of the vault when a
// module here asks for one (ADR 0550 §2).
vaultAuthority: trust.VaultAuthority,
certify: func(ctx context.Context) error {
return certifyThisNode(ctx, opts.state, node, trust, session.AskVault, func() { sayFacts(ctx, ag) })
},
}
// And its answer to every placement candidate, placed or not, applying nothing (novox/hq ADR 0559 §4): written into
// its own key, `<node>.node`, under candidates.
shadowHooks := hooks
shadowHooks.refused = nil
shadowHooks.refused, shadowHooks.certify = nil, nil
shadowEnv := composerEnv(session, node, say, shadowHooks)
// And, after each answer, what runs of an own secret the answer says the vault holds another value of, handed to
// the vault while the node applies the controller's compositions (issue 1126, cause C4; ownresync).
@@ -725,8 +742,19 @@ func busyModules() []string {
return out
}
// keptDeclaration is the declaration the node kept before the switch, verified with the mesh's key when it is given.
// keptRetired says the kept declaration is read no more: every module the catalogue places on this node has a have of
// its own (placedAllHave), so what runs is the agent's alone (ADR 0550 §2, design 47's L3 s9).
var keptRetired atomic.Bool
// errKeptRetired is keptDeclaration's answer once the read is retired.
var errKeptRetired = errors.New("the kept declaration is read no more: every module placed on this node has a have of its own")
// keptDeclaration is the declaration the node kept before the switch, verified with the mesh's key when it is given;
// once the read is retired, nothing is read.
func keptDeclaration(statePath string, signer ed25519.PublicKey) (*declaration.Declaration, error) {
if keptRetired.Load() {
return nil, errKeptRetired
}
var raw []byte
if len(signer) > 0 {
var err error
@@ -903,6 +931,9 @@ type composerHooks struct {
unseal func(string) ([]byte, error)
refused func(why string, held map[string]string)
held func(module string) (delivery.Seals, []string, error)
// vaultAuthority and certify are selfcompose.Env's VaultAuthority and Certify.
vaultAuthority func() string
certify func(ctx context.Context) error
}
func selfComposer(ctx context.Context, session *link.Session, node string, say func(string), hooks composerHooks) (*selfcompose.Composer, string) {
@@ -966,7 +997,8 @@ func composerEnv(session *link.Session, node string, say func(string), hooks com
Hear: session.Hear,
// And a moved module's templates archive, in the artifact store, as every archive is fetched (issue 1126).
Fetch: apply.Fetch,
Running: hooks.running, Unseal: hooks.unseal, Refused: hooks.refused, Held: hooks.held}
Running: hooks.running, Unseal: hooks.unseal, Refused: hooks.refused, Held: hooks.held,
VaultAuthority: hooks.vaultAuthority, Certify: hooks.certify}
}
// endWhenUnplaced calls stop once this node's placement key is deleted from the seat's placement (after the watch
+2 -1
View File
@@ -159,7 +159,8 @@ func (b *itBed) start(trustOperator bool) (stop func()) {
done := make(chan struct{})
go func() {
defer close(done)
nodeAgent(ctx, b.opts, b.controll.Public().(ed25519.PublicKey), b.session, nil, b.say, func() {},
controller := b.controll.Public().(ed25519.PublicKey)
nodeAgent(ctx, b.opts, controller, newMeshTrust(controller), b.session, nil, b.say, func() {},
&agentNow{}, func(context.Context, *agent.Agent) {})
}()
return func() { cancel(); <-done }
+307
View File
@@ -0,0 +1,307 @@
package main
import (
"context"
"crypto/ed25519"
"encoding/base64"
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"time"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/overlay"
"github.com/novox/mesh-host/internal/identity"
"github.com/novox/mesh-host/internal/link"
)
// **What this engine trusts a mesh signature from** (novox/hq ADR 0550 §2; design 47's L3 item d). Two signers: the
// controller's key, which the node was enrolled with (its membership's signer), and the vault's key, read from the vault's
// state `signers`, which only the vault may write: the bus grants are its anchor, as the wants' and the placement's (the
// lead's design limit, 2026-10-28). The controller's key is trusted until its lease is handed away, heard on the lease key
// (buscompose.LeaseHandedAway) and never on a clock; from then the vault's alone. What it trusts now the node states in its
// record (NodeRecord.Trusts), so the gate before the lease write reads every node's from the bus.
type meshTrust struct {
mu sync.Mutex
controller ed25519.PublicKey
handedAway bool
vault ed25519.PublicKey
authority string
// changed is told when what it trusts changed, so the record is written again.
changed func()
}
// newMeshTrust trusts the controller's key alone until the vault's is read.
func newMeshTrust(controller ed25519.PublicKey) *meshTrust {
t := &meshTrust{}
if len(controller) == ed25519.PublicKeySize {
t.controller = controller
}
return t
}
// onChange names who is told when what it trusts changed.
func (t *meshTrust) onChange(f func()) {
t.mu.Lock()
t.changed = f
t.mu.Unlock()
}
// Controller is the controller's key while the controller holds its lease, and nil once the lease is handed away.
func (t *meshTrust) Controller() ed25519.PublicKey {
if t == nil {
return nil
}
t.mu.Lock()
defer t.mu.Unlock()
if t.handedAway {
return nil
}
return t.controller
}
// Signers are the keys a mesh signature is verified against now: the controller's while it holds the lease, and the
// vault's once read.
func (t *meshTrust) Signers() []delivery.Signer {
t.mu.Lock()
defer t.mu.Unlock()
var out []delivery.Signer
if t.controller != nil && !t.handedAway {
out = append(out, delivery.Signer{Name: delivery.SignerController, Key: t.controller})
}
if t.vault != nil {
out = append(out, delivery.Signer{Name: delivery.SignerVault, Key: t.vault})
}
return out
}
// Verify is the signer that signed msg as kind, or why none it trusts did.
func (t *meshTrust) Verify(kind string, msg, sig []byte) (delivery.Signer, error) {
context, err := delivery.SignContext(kind)
if err != nil {
return delivery.Signer{}, err
}
return delivery.VerifyBy(t.Signers(), context, msg, sig)
}
// VaultAuthority is the vault's authority certificate as read from `signers`, or "".
func (t *meshTrust) VaultAuthority() string {
if t == nil {
return ""
}
t.mu.Lock()
defer t.mu.Unlock()
return t.authority
}
// Trusts is what the record says it trusts: each signer's key and the vault's authority, by fingerprint.
func (t *meshTrust) Trusts() []string {
if t == nil {
return nil
}
authorities := map[string]string{}
if a := t.VaultAuthority(); a != "" {
authorities[delivery.SignerVault] = a
}
return delivery.Trusts(t.Signers(), authorities)
}
// change applies f under the lock and tells who follows when Trusts differs after it.
func (t *meshTrust) change(f func()) {
before := strings.Join(t.Trusts(), "\n")
t.mu.Lock()
f()
changed := t.changed
t.mu.Unlock()
if strings.Join(t.Trusts(), "\n") != before && changed != nil {
changed()
}
}
// heardSigner takes one key of the vault's `signers`: its key or its authority, refused (and said) when it does not
// check; a deleted key is no longer trusted.
func (t *meshTrust) heardSigner(key string, value []byte, deleted bool, say link.Announce) {
switch key {
case delivery.SignersKeyKey:
var pub ed25519.PublicKey
if !deleted && len(value) > 0 {
k, err := delivery.DecodeSignerKey(value)
if err == nil {
pub, err = k.PublicKey()
}
if err != nil {
say("the vault's signing key in " + delivery.SignersRead + " is not trusted: " + err.Error())
return
}
}
t.change(func() { t.vault = pub })
case delivery.SignersAuthorityKey:
pem := ""
if !deleted && len(value) > 0 {
a, err := delivery.DecodeSignerAuthority(value)
if err != nil {
say("the vault's authority in " + delivery.SignersRead + " is not trusted: " + err.Error())
return
}
pem = a.Certificate
}
t.change(func() { t.authority = pem })
}
}
// heardLease takes the controller's lease key as read: once handed away to a holder that is no controller, the
// controller's key is trusted no more. A lease that cannot be read changes nothing.
func (t *meshTrust) heardLease(raw []byte, found bool, readErr error) {
if readErr != nil {
return
}
handed, _ := buscompose.LeaseHandedAway(raw, found, nil)
t.change(func() { t.handedAway = handed })
}
// trustFollowAgain bounds the wait before a watch that could not be made is tried again on the same link: the vault may
// not serve `sign` yet, and until it does this node is not granted the reads.
var trustFollowAgain = 2 * time.Minute
// followTrust follows the vault's `signers` and the controller's lease on every link, as watches (never a poll), for as
// long as ctx lasts. A watch that cannot be made is said once per reason and tried again.
func followTrust(ctx context.Context, session *link.Session, trust *meshTrust, say link.Announce) {
said := ""
sayOnce := func(line string) {
if line != said {
said = line
if line != "" {
say(line)
}
}
}
for ctx.Err() == nil {
linked, cancel, err := session.WhenLinked(ctx)
if err != nil {
return
}
watching, stop := context.WithCancel(linked)
signers, serr := session.KVWatch(watching, delivery.SignersBucket, ">")
lease, lerr := session.KVWatch(watching, buscompose.LeaseBucket, buscompose.ControllerLeaseKey)
if serr != nil || lerr != nil {
stop()
sayOnce(fmt.Sprintf("what this node trusts a mesh signature from is not followed yet (the vault's signers: %v; "+
"the controller's lease: %v): it trusts what it last read, and looks again in %s", serr, lerr, trustFollowAgain))
signers, lease = nil, nil
} else {
sayOnce("")
}
for signers != nil || lease != nil {
select {
case u, ok := <-signers:
if !ok {
signers = nil
continue
}
if !u.Caught {
trust.heardSigner(u.Key, u.Value, u.Deleted, say)
}
case u, ok := <-lease:
if !ok {
lease = nil
continue
}
if !u.Caught {
trust.heardLease(u.Value, !u.Deleted && len(u.Value) > 0, nil)
}
}
}
stop()
select {
case <-linked.Done():
case <-time.After(trustFollowAgain):
}
cancel()
}
}
// placedAllHave says whether every module the catalogue places on node has a have of its own, read from the bus: then
// the kept declaration is read no more (keptRetired). A node the catalogue does not place, or whose placement or haves
// cannot be read, keeps reading it.
func placedAllHave(ctx context.Context, session *link.Session, node string) (bool, string) {
key, err := delivery.PlacementKey(node)
if err != nil {
return false, err.Error()
}
h, err := session.KVHistory(ctx, delivery.PlacementBucket, key)
if err != nil || len(h) == 0 || h[len(h)-1].Deleted {
return false, "its placement is not read"
}
p, err := delivery.DecodePlacement(h[len(h)-1].Value)
if err != nil {
return false, err.Error()
}
return allHave(p, func(module string) (bool, error) {
_, ok, err := session.ReadHaveErr(ctx, module)
return ok, err
})
}
// allHave says whether has finds a have for every module p places, and why not.
func allHave(p delivery.Placement, has func(module string) (bool, error)) (bool, string) {
if len(p.Modules) == 0 {
return false, "the catalogue places no module here"
}
for _, m := range p.Modules {
ok, err := has(m.Module)
if err != nil {
return false, "the have of " + m.Module + " is not read: " + err.Error()
}
if !ok {
return false, m.Module + " has no have yet"
}
}
return true, fmt.Sprintf("each of the %d modules placed here has a have of its own", len(p.Modules))
}
// certifyThisNode asks the vault's `sign` for this node's own certificate (kind node-certificate, ADR 0550 §2), checks
// it is for this node's internal name and serving key and signed by the authority the answer names, which must be the
// vault's this node trusts, keeps it where the record reads it, and writes the record again (written).
func certifyThisNode(ctx context.Context, state, node string, trust *meshTrust,
ask func(ctx context.Context, verb string, args any) (json.RawMessage, error), written func()) error {
serving, err := identity.LoadServingKey(identity.ServingKeyPath(state))
if err != nil {
return err
}
key, err := base64.StdEncoding.DecodeString(serving.Public)
if err != nil || len(key) != ed25519.PublicKeySize {
return fmt.Errorf("this node's serving key is not an ed25519 key")
}
raw, err := ask(ctx, delivery.SignVerb, map[string]any{"kind": delivery.KindNodeCertificate, "node": node})
if err != nil {
return err
}
var a delivery.SignAnswer
if err := json.Unmarshal(raw, &a); err != nil {
return fmt.Errorf("the vault's answer is not a signing: %w", err)
}
trusted := trust.VaultAuthority()
if trusted == "" {
return fmt.Errorf("this node has read no authority of the vault's to check the certificate against")
}
if err := delivery.CheckNodeCertificate(a.Certificate, []string{trusted}, overlay.InternalName(node),
ed25519.PublicKey(key), time.Now()); err != nil {
return err
}
path := identity.CertificatePath(state)
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return err
}
if err := os.WriteFile(path, []byte(a.Certificate), 0o644); err != nil {
return err
}
if written != nil {
written()
}
return nil
}
+37
View File
@@ -4,6 +4,7 @@ import (
"crypto/ed25519"
"encoding/json"
"errors"
"path/filepath"
"strings"
"testing"
@@ -11,6 +12,7 @@ import (
"git.novox.be/novox/mesh-sdk/go/delivery"
"github.com/novox/mesh-host/internal/agent"
"github.com/novox/mesh-host/internal/store"
)
func aFileOf(module, id string) json.RawMessage {
@@ -59,6 +61,41 @@ func TestDeclaredNowReadsWhatRunsWhenTheAgentIsRunning(t *testing.T) {
if err != nil || d.Provisioner != "from-agent" {
t.Fatalf("declaredNow read %+v (%v), not what runs", d, err)
}
// L3 s9 (ADR 0550 §2): a node whose placed modules all have a have reads the kept declaration no more, though the
// file is there and readable and the controller's key is dropped; until then it reads it.
state := filepath.Join(t.TempDir(), "state", "state.json")
if err := store.SaveDeclared(store.DeclaredPath(state), store.Declared{Declaration: []byte(`{"declaration":1,` +
`"resources":[{"id":"zsh.rc","type":"file","module":"zsh","path":"/etc/zshrc","content":"x"}]}`)}); err != nil {
t.Fatal(err)
}
if d, err := keptDeclaration(state, nil); err != nil || len(d.Resources) != 1 {
t.Fatalf("before the read is retired: %+v, %v", d, err)
}
placed := delivery.Placement{Node: "n1", Modules: []delivery.PlacedModule{{Module: "zsh"}, {Module: "lemurs"}}}
for _, row := range []struct {
name string
has map[string]bool
want bool
}{
{"every placed module has a have", map[string]bool{"zsh": true, "lemurs": true}, true},
{"one has none yet", map[string]bool{"zsh": true}, false},
} {
if got, why := allHave(placed, func(m string) (bool, error) { return row.has[m], nil }); got != row.want {
t.Errorf("%s: %v (%s)", row.name, got, why)
}
}
if got, _ := allHave(delivery.Placement{Node: "n1"}, nil); got {
t.Error("a node with nothing placed retires the read")
}
keptRetired.Store(true)
defer keptRetired.Store(false)
if _, err := keptDeclaration(state, nil); !errors.Is(err, errKeptRetired) {
t.Errorf("after the read is retired: %v", err)
}
if by, filled := keptOnce(options{state: state}, nil)(); by != nil || filled != nil {
t.Errorf("after the read is retired the kept declaration was split: %v %v", by, filled)
}
}
// 1945/AC17 (unit): a resource the node's own composition shares with a running module's composition is declared once,
+3 -2
View File
@@ -274,7 +274,8 @@ func TestTheReportIsSentFirstAndTheNodeKeyCarriesTheNodesFacts(t *testing.T) {
// 1126 (ADR 0555 §3): with its keys, the node key carries the node's record: the profile's present capabilities and
// networks, the outward links, the health's contract and the public keys; a report that changes none of it writes
// nothing, a changed fact writes the key once, and a report without a profile or health leaves them standing.
keys := RecordKeys{TunnelKey: "tk", SealingKey: "sk", ServingKey: "pk", Certificate: "cert"}
keys := RecordKeys{TunnelKey: "tk", SealingKey: "sk", ServingKey: "pk", Certificate: "cert",
Trusts: []string{"controller key sha256:aa", "vault key sha256:bb"}} // and what it trusts (L3 s6, ADR 0550 §2)
q.Keys = func() RecordKeys { return keys }
rep.Profile = map[string]any{"capabilities": []any{map[string]any{"name": "systemd", "present": true},
map[string]any{"name": "docker", "present": false}, map[string]any{"name": "apt", "present": true}},
@@ -286,7 +287,7 @@ func TestTheReportIsSentFirstAndTheNodeKeyCarriesTheNodesFacts(t *testing.T) {
t.Fatalf("the node key holds no record: %+v, %v", got, err)
}
wantRecord := delivery.NodeRecord{V: delivery.Version, Node: "n1", TunnelKey: "tk", SealingKey: "sk", ServingKey: "pk",
Certificate: "cert", Capabilities: []string{"apt", "systemd"}, OutwardLinks: []string{"eth0"}, NetworksSaid: true,
Certificate: "cert", Trusts: keys.Trusts, Capabilities: []string{"apt", "systemd"}, OutwardLinks: []string{"eth0"}, NetworksSaid: true,
Networks: []catalogue.AttachedNetwork{{Link: "eth0", Network: "192.0.2.0/24"}}, Contract: NodeKeyContract}
if !delivery.SameNodeRecord(*got.Record, wantRecord) {
t.Errorf("record = %+v, want %+v", *got.Record, wantRecord)
+3 -1
View File
@@ -23,6 +23,8 @@ type RecordKeys struct {
Certificate string
// Authority is the mesh's own certificate as this machine last applied it (public).
Authority string
// Trusts is what this machine trusts a mesh signature from now (delivery.Trusts, ADR 0550 §2).
Trusts []string
}
// recordOf is the node's record from one facts report and its keys; nil for a report that carries no profile (one that
@@ -45,7 +47,7 @@ func recordOf(node string, r Report, keys RecordKeys) *delivery.NodeRecord {
return nil
}
rec := &delivery.NodeRecord{V: delivery.Version, Node: node, TunnelKey: keys.TunnelKey, SealingKey: keys.SealingKey,
ServingKey: keys.ServingKey, Certificate: keys.Certificate, Authority: keys.Authority}
ServingKey: keys.ServingKey, Certificate: keys.Certificate, Authority: keys.Authority, Trusts: keys.Trusts}
for _, c := range p.Capabilities {
if c.Present && c.Name != "" {
rec.Capabilities = append(rec.Capabilities, c.Name)
+77
View File
@@ -12,7 +12,9 @@ package selfcompose
import (
"context"
"crypto/ed25519"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
@@ -27,6 +29,7 @@ import (
"git.novox.be/novox/mesh-sdk/go/compose"
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/overlay"
"github.com/nats-io/nats.go"
@@ -74,6 +77,13 @@ type Env struct {
// Fetch reads an archive at a URL in the artifact store: a moved module's templates archive, whose texts its built
// manifest carries (issue 1126). Read once per digest. Nil reads none, and a moved module does not compose.
Fetch func(ctx context.Context, url string) ([]byte, error)
// VaultAuthority is the vault's authority as this node read it from the vault's `signers` (ADR 0550 §2), composed
// beside the controller's as the mesh's authority (frombus.SignersReader); "" or nil composes the controller's alone.
VaultAuthority func() string
// Certify asks the vault's `sign` for this node's own certificate, keeps it and writes the node's record again (ADR
// 0550 §2): asked only when a module placed here asks for a certificate and the record holds none from an authority
// the node trusts. Nil asks nothing, and such a module stands in, as before.
Certify func(ctx context.Context) error
}
// VaultSecretChanged is the vault's event that a secret changed: a name and its generation, never a value.
@@ -98,6 +108,8 @@ type Composer struct {
// sealedOwn is each own secret's sealed value as first handed, by module/name and generation: a value sealed again
// is other bytes, so the one first had is kept while the generation stands, and the composition stays the same.
sealedOwn map[string]ownSealed
// certified is the serving key a certificate was last asked for successfully, so it is asked once per key.
certified string
said string
refusal string
// doing is what the composer waits on now (waiting), for a running answer to say (issue 1126, T2); its own lock, so
@@ -497,6 +509,8 @@ func (c *Composer) compose(ctx context.Context) []agent.Update {
if c.env.Running != nil {
running = c.env.Running()
}
// Its own certificate, asked of the vault when a module here asks for one and the record holds none it trusts.
c.certifyIfWanted(ctx, r, own)
scope := frombus.Scope{Node: c.env.Node, KeptAt: kept, InOwn: inOwn, Running: running}
missed := c.missed.Load()
compositions, _, held, err := frombus.Compose(ctx, r, c.env.Mesh, scope, fps)
@@ -731,6 +745,69 @@ func (c *Composer) Policy(ctx context.Context, module string) string {
// reader is the composer's view of the bus as frombus reads it.
type reader struct{ c *Composer }
// VaultAuthority is the vault's authority the node follows (frombus.SignersReader).
func (r reader) VaultAuthority(context.Context) string {
if r.c.env.VaultAuthority == nil {
return ""
}
return r.c.env.VaultAuthority()
}
var _ frombus.SignersReader = reader{}
// certificateWanted says whether a module the placement p puts on this node asks for a certificate for the machine's
// internal name (its manifest's `certificate`), read from the indexer's manifests; a manifest that cannot be read is
// taken as asking none, and the composition says what it misses.
func (c *Composer) certificateWanted(ctx context.Context, r reader, p delivery.Placement) bool {
for _, m := range p.Modules {
c.mu.Lock()
digest := c.modules[m.Module].Manifest
c.mu.Unlock()
if digest == "" {
continue
}
raw, err := r.Manifest(ctx, digest)
if err != nil {
continue
}
if parsed, err := catalogue.ParseManifestForInputs(raw); err == nil && parsed.Certificate != nil {
return true
}
}
return false
}
// certifyIfWanted asks Certify once per serving key when a module placed here asks for a certificate and this node's
// record holds none for its internal name and serving key from an authority it trusts (the one it holds, or the vault's).
func (c *Composer) certifyIfWanted(ctx context.Context, r reader, p delivery.Placement) {
if c.env.Certify == nil || !c.certificateWanted(ctx, r, p) {
return
}
c.mu.Lock()
rec, ok := c.records[c.env.Node]
asked := c.certified
c.mu.Unlock()
if !ok || rec.ServingKey == "" || asked == rec.ServingKey {
return
}
key, err := base64.StdEncoding.DecodeString(rec.ServingKey)
if err != nil || len(key) != ed25519.PublicKeySize {
return
}
trusted := []string{rec.Authority, r.VaultAuthority(ctx)}
if rec.Certificate != "" && delivery.CheckNodeCertificate(rec.Certificate, trusted, overlay.InternalName(c.env.Node),
ed25519.PublicKey(key), time.Now()) == nil {
return
}
if err := c.env.Certify(ctx); err != nil {
c.sayOnce(fmt.Sprintf("a module here asks for a certificate, and the vault's sign gave none: %v", err))
return
}
c.mu.Lock()
c.certified = rec.ServingKey
c.mu.Unlock()
}
func (r reader) Placements(context.Context) (map[string]delivery.Placement, error) {
r.c.mu.Lock()
defer r.c.mu.Unlock()
+27
View File
@@ -3,6 +3,7 @@ package selfcompose
import (
"context"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"fmt"
@@ -1219,6 +1220,32 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
}
}
b.mu.Unlock()
// L3 s7 (ADR 0550 §2): the node asks the vault for its own certificate only when a module placed here asks for one
// and its record holds none from an authority it trusts; once per serving key.
certBus := newFakeBus()
certAsks := 0
certEnv := certBus.env()
certEnv.Certify = func(context.Context) error { certAsks++; return nil }
certEnv.VaultAuthority = func() string { return "" }
cc := New(certEnv)
withCert := `{"module":"web","version":"1","certificate":{"into":"/etc/web/tls.crt"}}`
cdigest := delivery.ManifestDigest([]byte(withCert))
cc.manifests[digest] = []byte(lemurs)
cc.manifests[cdigest] = []byte(withCert)
cc.modules = map[string]delivery.Module{"lemurs": {Manifest: digest}, "web": {Manifest: cdigest}}
cc.records["n1"] = delivery.NodeRecord{Node: "n1", ServingKey: base64.StdEncoding.EncodeToString(make([]byte, 32))}
onlyLemurs := delivery.Placement{Node: "n1", Modules: []delivery.PlacedModule{{Module: "lemurs"}}}
cc.certifyIfWanted(context.Background(), reader{c: cc}, onlyLemurs)
if certAsks != 0 {
t.Errorf("no module asks for a certificate, and the vault was asked %d times", certAsks)
}
withWeb := delivery.Placement{Node: "n1", Modules: []delivery.PlacedModule{{Module: "lemurs"}, {Module: "web"}}}
cc.certifyIfWanted(context.Background(), reader{c: cc}, withWeb)
cc.certifyIfWanted(context.Background(), reader{c: cc}, withWeb)
if certAsks != 1 {
t.Errorf("a module asks for a certificate: the vault was asked %d times, want once", certAsks)
}
}
// patternBus is a bus whose every watch gets the updates of its bucket matching its pattern, as the bus's own watches