From 479ac71050b40838e8740f68f2fedc8e5e93e538 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 28 Oct 2026 09:34:53 +0100 Subject: [PATCH] =?UTF-8?q?The=20node=20trusts=20the=20vault's=20key=20bes?= =?UTF-8?q?ide=20the=20controller's=20until=20the=20lease=20is=20handed=20?= =?UTF-8?q?away,=20says=20what=20it=20trusts=20in=20its=20record,=20asks?= =?UTF-8?q?=20the=20vault=20for=20its=20own=20certificate,=20composes=20th?= =?UTF-8?q?e=20vault's=20authority=20beside=20the=20controller's,=20and=20?= =?UTF-8?q?reads=20the=20kept=20declaration=20no=20more=20once=20every=20p?= =?UTF-8?q?laced=20module=20has=20a=20have=20(issue=20774,=20ADR=200550=20?= =?UTF-8?q?=C2=A72)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/mesh-host/authority_test.go | 113 ++++++- cmd/mesh-host/main.go | 15 +- cmd/mesh-host/nodeagent.go | 56 +++- cmd/mesh-host/nodeagent_integration_test.go | 3 +- cmd/mesh-host/trust.go | 307 ++++++++++++++++++++ cmd/mesh-host/whatruns_test.go | 37 +++ internal/link/have_test.go | 5 +- internal/link/record.go | 4 +- internal/selfcompose/selfcompose.go | 77 +++++ internal/selfcompose/selfcompose_test.go | 27 ++ 10 files changed, 623 insertions(+), 21 deletions(-) create mode 100644 cmd/mesh-host/trust.go diff --git a/cmd/mesh-host/authority_test.go b/cmd/mesh-host/authority_test.go index 48ce708..84d8ea9 100644 --- a/cmd/mesh-host/authority_test.go +++ b/cmd/mesh-host/authority_test.go @@ -1,16 +1,27 @@ package main import ( + "context" + "crypto/ed25519" + "crypto/x509" + "encoding/base64" "encoding/json" + "encoding/pem" "os" "path/filepath" "testing" + "time" + "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/overlay" + + "github.com/novox/mesh-host/internal/identity" ) // Issue 1126 (ADR 0555 §3): the mesh's own certificate a composition carries is kept for the machine's record, and the -// record's keys read it; a composition carrying none changes nothing. +// record's keys read it; a composition carrying none changes nothing. And (L3 s6, s7; ADR 0550 §2) the record says what +// the node trusts a mesh signature from, and the node keeps the certificate the vault signed for it. func TestTheAppliedAuthorityIsKeptForTheRecord(t *testing.T) { state := filepath.Join(t.TempDir(), "state", "state.json") res := func(id, content string) json.RawMessage { @@ -26,7 +37,105 @@ func TestTheAppliedAuthorityIsKeptForTheRecord(t *testing.T) { if err := keepAuthority(state, delivery.Composed{Resources: []json.RawMessage{res("ca-trust.certificate-authority", "CA-1")}}); err != nil { t.Fatal(err) } - if got := recordKeys(state)(); got.Authority != "CA-1" { + if got := recordKeys(state, nil)(); got.Authority != "CA-1" { t.Errorf("the record's keys read the authority %q", got.Authority) } + + // L3 s6: before the vault's signers are read the node trusts the controller's key alone; once read, a message the + // vault signed is accepted and the record names both; once the lease is handed away the controller's key is refused + // and the record names the vault's alone. Each change tells the record's writer. + controllerPub, controller, _ := ed25519.GenerateKey(nil) + vaultPub, vault, _ := ed25519.GenerateKey(nil) + trust := newMeshTrust(controllerPub) + told := 0 + trust.onChange(func() { told++ }) + keys := recordKeys(state, trust.Trusts) + ctrlFP, vaultFP := delivery.SignerFingerprint(controllerPub), delivery.SignerFingerprint(vaultPub) + if got := keys().Trusts; !delivery.TrustsKey(got, delivery.SignerController, ctrlFP) || delivery.TrustsKey(got, delivery.SignerVault, vaultFP) { + t.Errorf("before the signers are read the record trusts %v", got) + } + enrol, _ := delivery.SignContext(delivery.KindEnrolment) + msg := []byte("a reply") + if _, err := trust.Verify(delivery.KindEnrolment, msg, delivery.SignMessage(vault, enrol, msg)); err == nil { + t.Error("a vault-signed message is accepted before the vault's key is read") + } + vaultCA := testAuthority(t, vaultPub, vault) + keyValue, _ := json.Marshal(delivery.NewSignerKey(vaultPub, time.Now())) + fp, _ := delivery.AuthorityFingerprint(vaultCA) + authValue, _ := json.Marshal(delivery.SignerAuthority{Certificate: vaultCA, Fingerprint: fp, Made: time.Now()}) + trust.heardSigner(delivery.SignersKeyKey, keyValue, false, func(string) {}) + trust.heardSigner(delivery.SignersAuthorityKey, authValue, false, func(string) {}) + if s, err := trust.Verify(delivery.KindEnrolment, msg, delivery.SignMessage(vault, enrol, msg)); err != nil || s.Name != delivery.SignerVault { + t.Errorf("a vault-signed message on a node that read signers: %+v, %v", s, err) + } + if got := keys().Trusts; !delivery.TrustsKey(got, delivery.SignerController, ctrlFP) || !delivery.TrustsKey(got, delivery.SignerVault, vaultFP) || + trust.VaultAuthority() != vaultCA || told != 2 { + t.Errorf("after the signers are read the record trusts %v (told %d)", got, told) + } + handed, _ := json.Marshal(buscompose.LeaseHolder{HandedTo: "mesh-vault", HandedBy: "person.operator"}) + trust.heardLease(handed, true, nil) + if _, err := trust.Verify(delivery.KindEnrolment, msg, delivery.SignMessage(controller, enrol, msg)); err == nil || trust.Controller() != nil { + t.Error("after the lease is handed away the controller's key is still trusted") + } + if got := keys().Trusts; delivery.TrustsKey(got, delivery.SignerController, ctrlFP) || !delivery.TrustsKey(got, delivery.SignerVault, vaultFP) || told != 3 { + t.Errorf("after the lease is handed away the record trusts %v (told %d)", got, told) + } + trust.heardLease(handed, true, nil) + if told != 3 { + t.Error("a lease heard again unchanged wrote the record again") + } + // A signers' key naming another fingerprint is not trusted. + bad, _ := json.Marshal(delivery.SignerKey{Key: base64.StdEncoding.EncodeToString(controllerPub), Fingerprint: vaultFP}) + trust.heardSigner(delivery.SignersKeyKey, bad, false, func(string) {}) + if !delivery.TrustsKey(keys().Trusts, delivery.SignerVault, vaultFP) { + t.Error("a signers' key that does not check replaced the vault's") + } + + // L3 s7: the certificate the vault signs for this node's internal name and serving key is kept where the record reads + // it, and the record is written again; one signed by another authority is refused and nothing is kept. + serving, err := identity.GenerateServingKey() + if err != nil { + t.Fatal(err) + } + if err := identity.WriteServingKey(identity.ServingKeyPath(state), serving); err != nil { + t.Fatal(err) + } + servingPub, _ := base64.StdEncoding.DecodeString(serving.Public) + signedBy := func(ca string, key ed25519.PrivateKey) string { + parent, _ := delivery.ParseCertificate(ca) + der, err := x509.CreateCertificate(nil, delivery.NodeCertificateTemplate(overlay.InternalName("n1"), servingPub, time.Now().Add(-time.Hour)), + parent, ed25519.PublicKey(servingPub), key) + if err != nil { + t.Fatal(err) + } + return string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})) + } + var asked []any + answer := signedBy(vaultCA, vault) + ask := func(_ context.Context, verb string, args any) (json.RawMessage, error) { + asked = append(asked, verb, args) + return json.Marshal(delivery.SignAnswer{Kind: delivery.KindNodeCertificate, Node: "n1", Certificate: answer}) + } + written := 0 + if err := certifyThisNode(context.Background(), state, "n1", trust, ask, func() { written++ }); err != nil { + t.Fatal(err) + } + if got := recordKeys(state, nil)(); got.Certificate != answer || written != 1 || len(asked) != 2 || asked[0] != delivery.SignVerb { + t.Errorf("the vault's certificate: kept %v, written %d, asked %v", got.Certificate == answer, written, asked) + } + answer = signedBy(testAuthority(t, controllerPub, controller), controller) + if err := certifyThisNode(context.Background(), state, "n1", trust, ask, func() { written++ }); err == nil || written != 1 { + t.Errorf("a certificate of another authority was kept: %v", err) + } +} + +// testAuthority is a self-signed authority certificate for key, as the vault makes its own. +func testAuthority(t *testing.T, pub ed25519.PublicKey, key ed25519.PrivateKey) string { + t.Helper() + tpl := delivery.AuthorityTemplate(pub, time.Now().Add(-time.Hour)) + der, err := x509.CreateCertificate(nil, tpl, tpl, pub, key) + if err != nil { + t.Fatal(err) + } + return string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})) } diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index d8b3610..e0696c1 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -1105,10 +1105,13 @@ func runLink(ctx context.Context, opts options) error { // the bus, answer what the controller asks of this machine, and say its facts: when a link opens, and when the // controller asks. proof := &proving{statePath: opts.state, version: runningVersion(), say: say} + // What this engine trusts a mesh signature from: the controller's key until its lease is handed away, and the vault's + // once read from its signers (novox/hq ADR 0550 §2). + trust := newMeshTrust(mine.Membership.Signer) queue := &link.Session{ Membership: membership, // And the public halves and certificate its record says (novox/hq ADR 0555 §3), read at each facts report. - Keys: recordKeys(opts.state), + Keys: recordKeys(opts.state, trust.Trusts), // And the first have this build writes is what proves it to the launcher (ADR 0539 §9, ADR 0227 §8). HaveWritten: proof.haveWritten, // And the operator's hand-over of a directory used as found, carried by the controller from the @@ -1173,7 +1176,10 @@ func runLink(ctx context.Context, opts options) error { } // The node follows its wants: one module at a time from the composition its want names (novox/hq ADR 0539), and the // busy containers are asked for again soon (ADR 0360 §11). - go nodeAgent(aside, opts, mine.Membership.Signer, queue, sched, say, afterApply, current, factsOfNow) + // And what it trusts, followed on the bus, said in its record whenever it changes (ADR 0550 §2). + trust.onChange(func() { factsOfNow(aside, current.get()) }) + go followTrust(aside, queue, trust, say) + go nodeAgent(aside, opts, mine.Membership.Signer, trust, queue, sched, say, afterApply, current, factsOfNow) go busyPasses(aside, func() { if ag := current.get(); ag != nil { ag.Retry(busyModules()...) @@ -2167,9 +2173,12 @@ func withMissing(st liveness.Statement) liveness.Statement { // recordKeys reads the public halves of the node's keys and the certificate the mesh issued for its internal name from // its state directory (novox/hq ADR 0555 §3): what the node's record says of them. A key that cannot be read is left // empty, never invented; the certificate is there only after the first declaration that carried it. -func recordKeys(state string) func() link.RecordKeys { +func recordKeys(state string, trusts func() []string) func() link.RecordKeys { return func() link.RecordKeys { var k link.RecordKeys + if trusts != nil { + k.Trusts = trusts() + } if id, err := identity.Load(identity.Path(state)); err == nil { k.TunnelKey = id.Overlay.Public } diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index 430be23..9452dc7 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -5,12 +5,14 @@ import ( "crypto/ed25519" "encoding/base64" "encoding/json" + "errors" "fmt" "os" "path/filepath" "sort" "strings" "sync" + "sync/atomic" "time" "git.novox.be/novox/mesh-sdk/go/compose/frombus" @@ -343,13 +345,14 @@ func resourcesOf(cs []delivery.Composed) []declaration.Resource { // controller's key the node was enrolled with (novox/hq ADR 0539 §3, §11); a node adopted later has it at its adoption. const operatorKeysFile = "operator-keys" -// trustedKeys are the keys a composition may be signed by: the controller's, which the node was enrolled with, and the -// operator's, read from operatorKeysFile each time it is asked, so a key that arrives while the agent runs is trusted at -// the next composition. A file that cannot be read trusts the controller's alone, and says so. -func trustedKeys(opts options, controller ed25519.PublicKey) func() []ed25519.PublicKey { +// trustedKeys are the keys a composition may be signed by: the controller's, which the node was enrolled with, while the +// controller holds its lease (ADR 0550 §2: trusted no more once it is handed away), and the operator's, read from +// operatorKeysFile each time it is asked, so a key that arrives while the agent runs is trusted at the next composition. +// A file that cannot be read trusts the controller's alone, and says so. +func trustedKeys(opts options, controllerNow func() ed25519.PublicKey) func() []ed25519.PublicKey { return func() []ed25519.PublicKey { keys := []ed25519.PublicKey{} - if len(controller) == ed25519.PublicKeySize { + if controller := controllerNow(); len(controller) == ed25519.PublicKeySize { keys = append(keys, controller) } raw, err := os.ReadFile(filepath.Join(filepath.Dir(opts.state), operatorKeysFile)) @@ -373,13 +376,16 @@ func trustedKeys(opts options, controller ed25519.PublicKey) func() []ed25519.Pu } // keptOnce reads the kept declaration once and splits it by module: its resources (the node's own are those that say -// none, ADR 0539 §9) and its filled paths. +// none, ADR 0539 §9) and its filled paths. Once the read is retired (keptRetired) nothing is read. func keptOnce(opts options, signer ed25519.PublicKey) func() (map[string][]json.RawMessage, map[string][]string) { var once sync.Once var by map[string][]json.RawMessage var filled map[string][]string return func() (map[string][]json.RawMessage, map[string][]string) { once.Do(func() { + if keptRetired.Load() { + return + } raw, err := store.LoadDeclared(store.DeclaredPath(opts.state), signer) if err != nil { return @@ -429,7 +435,7 @@ func splitKept(raw []byte) (map[string][]json.RawMessage, map[string][]string) { // nodeAgent runs the node agent under the session's links for as long as ctx lasts: it waits for a link, follows the // node's wants and compositions on it, and begins again with the next link. A watch it cannot make (a bucket this node's // account may not read) is said, and tried again with a growing wait, as the link is. -func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, session *link.Session, sched *apply.Scheduler, +func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trust *meshTrust, session *link.Session, sched *apply.Scheduler, say link.Announce, afterApply func(), current *agentNow, sayFacts func(ctx context.Context, ag *agent.Agent)) { wait := 2 * time.Second for ctx.Err() == nil { @@ -446,6 +452,11 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess return } var ag *agent.Agent + // The kept release-5 declaration is no longer read once every module the catalogue places here has a have of its + // own (ADR 0550 §2: its signature is the controller's, trusted no more after the lease write). + if retired, why := placedAllHave(linked, session, node); retired && !keptRetired.Swap(true) { + say("the kept declaration is no longer read: " + why) + } kept := keptOnce(opts, signer) // The mesh's own certificate the declaration kept from before carries, kept for the record before this machine // composes its own (ADR 0555 §3): without it a module that asks for a certificate would be stood in for. @@ -458,7 +469,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess } // The bus's user list, written alone from the newest composition of the bus module (ADR 0547 §8): on every // node, and a node whose compositions carry no list writes nothing. - users := userListFollower(opts, session, composedBucket, trustedKeys(opts, signer), say, func() { + users := userListFollower(opts, session, composedBucket, trustedKeys(opts, trust.Controller), say, func() { // This engine wrote the list, so the bus reloads its accounts and answers no JetStream request for a few // seconds: what failed in them is tried again once that is over, not at its next wait. time.AfterFunc(reloadSettles, func() { @@ -495,7 +506,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess } return out, nil }, - Trusted: trustedKeys(opts, signer), + Trusted: trustedKeys(opts, trust.Controller), Backup: backupNode(session), // What the filling refused before this apply may be named now: the provisioner's client is told to fill // again (ace, 2026-10-25 21:17). @@ -572,11 +583,17 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess now.NotComposedEach(run, why, held) // a module held alone marks that module, never the others (issue 1126) } }, + // And the vault's authority beside the controller's, and this node's certificate asked of the vault when a + // module here asks for one (ADR 0550 §2). + vaultAuthority: trust.VaultAuthority, + certify: func(ctx context.Context) error { + return certifyThisNode(ctx, opts.state, node, trust, session.AskVault, func() { sayFacts(ctx, ag) }) + }, } // And its answer to every placement candidate, placed or not, applying nothing (novox/hq ADR 0559 §4): written into // its own key, `.node`, under candidates. shadowHooks := hooks - shadowHooks.refused = nil + shadowHooks.refused, shadowHooks.certify = nil, nil shadowEnv := composerEnv(session, node, say, shadowHooks) // And, after each answer, what runs of an own secret the answer says the vault holds another value of, handed to // the vault while the node applies the controller's compositions (issue 1126, cause C4; ownresync). @@ -725,8 +742,19 @@ func busyModules() []string { return out } -// keptDeclaration is the declaration the node kept before the switch, verified with the mesh's key when it is given. +// keptRetired says the kept declaration is read no more: every module the catalogue places on this node has a have of +// its own (placedAllHave), so what runs is the agent's alone (ADR 0550 §2, design 47's L3 s9). +var keptRetired atomic.Bool + +// errKeptRetired is keptDeclaration's answer once the read is retired. +var errKeptRetired = errors.New("the kept declaration is read no more: every module placed on this node has a have of its own") + +// keptDeclaration is the declaration the node kept before the switch, verified with the mesh's key when it is given; +// once the read is retired, nothing is read. func keptDeclaration(statePath string, signer ed25519.PublicKey) (*declaration.Declaration, error) { + if keptRetired.Load() { + return nil, errKeptRetired + } var raw []byte if len(signer) > 0 { var err error @@ -903,6 +931,9 @@ type composerHooks struct { unseal func(string) ([]byte, error) refused func(why string, held map[string]string) held func(module string) (delivery.Seals, []string, error) + // vaultAuthority and certify are selfcompose.Env's VaultAuthority and Certify. + vaultAuthority func() string + certify func(ctx context.Context) error } func selfComposer(ctx context.Context, session *link.Session, node string, say func(string), hooks composerHooks) (*selfcompose.Composer, string) { @@ -966,7 +997,8 @@ func composerEnv(session *link.Session, node string, say func(string), hooks com Hear: session.Hear, // And a moved module's templates archive, in the artifact store, as every archive is fetched (issue 1126). Fetch: apply.Fetch, - Running: hooks.running, Unseal: hooks.unseal, Refused: hooks.refused, Held: hooks.held} + Running: hooks.running, Unseal: hooks.unseal, Refused: hooks.refused, Held: hooks.held, + VaultAuthority: hooks.vaultAuthority, Certify: hooks.certify} } // endWhenUnplaced calls stop once this node's placement key is deleted from the seat's placement (after the watch diff --git a/cmd/mesh-host/nodeagent_integration_test.go b/cmd/mesh-host/nodeagent_integration_test.go index b214352..852447b 100644 --- a/cmd/mesh-host/nodeagent_integration_test.go +++ b/cmd/mesh-host/nodeagent_integration_test.go @@ -159,7 +159,8 @@ func (b *itBed) start(trustOperator bool) (stop func()) { done := make(chan struct{}) go func() { defer close(done) - nodeAgent(ctx, b.opts, b.controll.Public().(ed25519.PublicKey), b.session, nil, b.say, func() {}, + controller := b.controll.Public().(ed25519.PublicKey) + nodeAgent(ctx, b.opts, controller, newMeshTrust(controller), b.session, nil, b.say, func() {}, &agentNow{}, func(context.Context, *agent.Agent) {}) }() return func() { cancel(); <-done } diff --git a/cmd/mesh-host/trust.go b/cmd/mesh-host/trust.go new file mode 100644 index 0000000..61d37fd --- /dev/null +++ b/cmd/mesh-host/trust.go @@ -0,0 +1,307 @@ +package main + +import ( + "context" + "crypto/ed25519" + "encoding/base64" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/overlay" + + "github.com/novox/mesh-host/internal/identity" + "github.com/novox/mesh-host/internal/link" +) + +// **What this engine trusts a mesh signature from** (novox/hq ADR 0550 §2; design 47's L3 item d). Two signers: the +// controller's key, which the node was enrolled with (its membership's signer), and the vault's key, read from the vault's +// state `signers`, which only the vault may write: the bus grants are its anchor, as the wants' and the placement's (the +// lead's design limit, 2026-10-28). The controller's key is trusted until its lease is handed away, heard on the lease key +// (buscompose.LeaseHandedAway) and never on a clock; from then the vault's alone. What it trusts now the node states in its +// record (NodeRecord.Trusts), so the gate before the lease write reads every node's from the bus. +type meshTrust struct { + mu sync.Mutex + controller ed25519.PublicKey + handedAway bool + vault ed25519.PublicKey + authority string + // changed is told when what it trusts changed, so the record is written again. + changed func() +} + +// newMeshTrust trusts the controller's key alone until the vault's is read. +func newMeshTrust(controller ed25519.PublicKey) *meshTrust { + t := &meshTrust{} + if len(controller) == ed25519.PublicKeySize { + t.controller = controller + } + return t +} + +// onChange names who is told when what it trusts changed. +func (t *meshTrust) onChange(f func()) { + t.mu.Lock() + t.changed = f + t.mu.Unlock() +} + +// Controller is the controller's key while the controller holds its lease, and nil once the lease is handed away. +func (t *meshTrust) Controller() ed25519.PublicKey { + if t == nil { + return nil + } + t.mu.Lock() + defer t.mu.Unlock() + if t.handedAway { + return nil + } + return t.controller +} + +// Signers are the keys a mesh signature is verified against now: the controller's while it holds the lease, and the +// vault's once read. +func (t *meshTrust) Signers() []delivery.Signer { + t.mu.Lock() + defer t.mu.Unlock() + var out []delivery.Signer + if t.controller != nil && !t.handedAway { + out = append(out, delivery.Signer{Name: delivery.SignerController, Key: t.controller}) + } + if t.vault != nil { + out = append(out, delivery.Signer{Name: delivery.SignerVault, Key: t.vault}) + } + return out +} + +// Verify is the signer that signed msg as kind, or why none it trusts did. +func (t *meshTrust) Verify(kind string, msg, sig []byte) (delivery.Signer, error) { + context, err := delivery.SignContext(kind) + if err != nil { + return delivery.Signer{}, err + } + return delivery.VerifyBy(t.Signers(), context, msg, sig) +} + +// VaultAuthority is the vault's authority certificate as read from `signers`, or "". +func (t *meshTrust) VaultAuthority() string { + if t == nil { + return "" + } + t.mu.Lock() + defer t.mu.Unlock() + return t.authority +} + +// Trusts is what the record says it trusts: each signer's key and the vault's authority, by fingerprint. +func (t *meshTrust) Trusts() []string { + if t == nil { + return nil + } + authorities := map[string]string{} + if a := t.VaultAuthority(); a != "" { + authorities[delivery.SignerVault] = a + } + return delivery.Trusts(t.Signers(), authorities) +} + +// change applies f under the lock and tells who follows when Trusts differs after it. +func (t *meshTrust) change(f func()) { + before := strings.Join(t.Trusts(), "\n") + t.mu.Lock() + f() + changed := t.changed + t.mu.Unlock() + if strings.Join(t.Trusts(), "\n") != before && changed != nil { + changed() + } +} + +// heardSigner takes one key of the vault's `signers`: its key or its authority, refused (and said) when it does not +// check; a deleted key is no longer trusted. +func (t *meshTrust) heardSigner(key string, value []byte, deleted bool, say link.Announce) { + switch key { + case delivery.SignersKeyKey: + var pub ed25519.PublicKey + if !deleted && len(value) > 0 { + k, err := delivery.DecodeSignerKey(value) + if err == nil { + pub, err = k.PublicKey() + } + if err != nil { + say("the vault's signing key in " + delivery.SignersRead + " is not trusted: " + err.Error()) + return + } + } + t.change(func() { t.vault = pub }) + case delivery.SignersAuthorityKey: + pem := "" + if !deleted && len(value) > 0 { + a, err := delivery.DecodeSignerAuthority(value) + if err != nil { + say("the vault's authority in " + delivery.SignersRead + " is not trusted: " + err.Error()) + return + } + pem = a.Certificate + } + t.change(func() { t.authority = pem }) + } +} + +// heardLease takes the controller's lease key as read: once handed away to a holder that is no controller, the +// controller's key is trusted no more. A lease that cannot be read changes nothing. +func (t *meshTrust) heardLease(raw []byte, found bool, readErr error) { + if readErr != nil { + return + } + handed, _ := buscompose.LeaseHandedAway(raw, found, nil) + t.change(func() { t.handedAway = handed }) +} + +// trustFollowAgain bounds the wait before a watch that could not be made is tried again on the same link: the vault may +// not serve `sign` yet, and until it does this node is not granted the reads. +var trustFollowAgain = 2 * time.Minute + +// followTrust follows the vault's `signers` and the controller's lease on every link, as watches (never a poll), for as +// long as ctx lasts. A watch that cannot be made is said once per reason and tried again. +func followTrust(ctx context.Context, session *link.Session, trust *meshTrust, say link.Announce) { + said := "" + sayOnce := func(line string) { + if line != said { + said = line + if line != "" { + say(line) + } + } + } + for ctx.Err() == nil { + linked, cancel, err := session.WhenLinked(ctx) + if err != nil { + return + } + watching, stop := context.WithCancel(linked) + signers, serr := session.KVWatch(watching, delivery.SignersBucket, ">") + lease, lerr := session.KVWatch(watching, buscompose.LeaseBucket, buscompose.ControllerLeaseKey) + if serr != nil || lerr != nil { + stop() + sayOnce(fmt.Sprintf("what this node trusts a mesh signature from is not followed yet (the vault's signers: %v; "+ + "the controller's lease: %v): it trusts what it last read, and looks again in %s", serr, lerr, trustFollowAgain)) + signers, lease = nil, nil + } else { + sayOnce("") + } + for signers != nil || lease != nil { + select { + case u, ok := <-signers: + if !ok { + signers = nil + continue + } + if !u.Caught { + trust.heardSigner(u.Key, u.Value, u.Deleted, say) + } + case u, ok := <-lease: + if !ok { + lease = nil + continue + } + if !u.Caught { + trust.heardLease(u.Value, !u.Deleted && len(u.Value) > 0, nil) + } + } + } + stop() + select { + case <-linked.Done(): + case <-time.After(trustFollowAgain): + } + cancel() + } +} + +// placedAllHave says whether every module the catalogue places on node has a have of its own, read from the bus: then +// the kept declaration is read no more (keptRetired). A node the catalogue does not place, or whose placement or haves +// cannot be read, keeps reading it. +func placedAllHave(ctx context.Context, session *link.Session, node string) (bool, string) { + key, err := delivery.PlacementKey(node) + if err != nil { + return false, err.Error() + } + h, err := session.KVHistory(ctx, delivery.PlacementBucket, key) + if err != nil || len(h) == 0 || h[len(h)-1].Deleted { + return false, "its placement is not read" + } + p, err := delivery.DecodePlacement(h[len(h)-1].Value) + if err != nil { + return false, err.Error() + } + return allHave(p, func(module string) (bool, error) { + _, ok, err := session.ReadHaveErr(ctx, module) + return ok, err + }) +} + +// allHave says whether has finds a have for every module p places, and why not. +func allHave(p delivery.Placement, has func(module string) (bool, error)) (bool, string) { + if len(p.Modules) == 0 { + return false, "the catalogue places no module here" + } + for _, m := range p.Modules { + ok, err := has(m.Module) + if err != nil { + return false, "the have of " + m.Module + " is not read: " + err.Error() + } + if !ok { + return false, m.Module + " has no have yet" + } + } + return true, fmt.Sprintf("each of the %d modules placed here has a have of its own", len(p.Modules)) +} + +// certifyThisNode asks the vault's `sign` for this node's own certificate (kind node-certificate, ADR 0550 §2), checks +// it is for this node's internal name and serving key and signed by the authority the answer names, which must be the +// vault's this node trusts, keeps it where the record reads it, and writes the record again (written). +func certifyThisNode(ctx context.Context, state, node string, trust *meshTrust, + ask func(ctx context.Context, verb string, args any) (json.RawMessage, error), written func()) error { + serving, err := identity.LoadServingKey(identity.ServingKeyPath(state)) + if err != nil { + return err + } + key, err := base64.StdEncoding.DecodeString(serving.Public) + if err != nil || len(key) != ed25519.PublicKeySize { + return fmt.Errorf("this node's serving key is not an ed25519 key") + } + raw, err := ask(ctx, delivery.SignVerb, map[string]any{"kind": delivery.KindNodeCertificate, "node": node}) + if err != nil { + return err + } + var a delivery.SignAnswer + if err := json.Unmarshal(raw, &a); err != nil { + return fmt.Errorf("the vault's answer is not a signing: %w", err) + } + trusted := trust.VaultAuthority() + if trusted == "" { + return fmt.Errorf("this node has read no authority of the vault's to check the certificate against") + } + if err := delivery.CheckNodeCertificate(a.Certificate, []string{trusted}, overlay.InternalName(node), + ed25519.PublicKey(key), time.Now()); err != nil { + return err + } + path := identity.CertificatePath(state) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + if err := os.WriteFile(path, []byte(a.Certificate), 0o644); err != nil { + return err + } + if written != nil { + written() + } + return nil +} diff --git a/cmd/mesh-host/whatruns_test.go b/cmd/mesh-host/whatruns_test.go index 2ffd7c6..1d4601e 100644 --- a/cmd/mesh-host/whatruns_test.go +++ b/cmd/mesh-host/whatruns_test.go @@ -4,6 +4,7 @@ import ( "crypto/ed25519" "encoding/json" "errors" + "path/filepath" "strings" "testing" @@ -11,6 +12,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/delivery" "github.com/novox/mesh-host/internal/agent" + "github.com/novox/mesh-host/internal/store" ) func aFileOf(module, id string) json.RawMessage { @@ -59,6 +61,41 @@ func TestDeclaredNowReadsWhatRunsWhenTheAgentIsRunning(t *testing.T) { if err != nil || d.Provisioner != "from-agent" { t.Fatalf("declaredNow read %+v (%v), not what runs", d, err) } + + // L3 s9 (ADR 0550 §2): a node whose placed modules all have a have reads the kept declaration no more, though the + // file is there and readable and the controller's key is dropped; until then it reads it. + state := filepath.Join(t.TempDir(), "state", "state.json") + if err := store.SaveDeclared(store.DeclaredPath(state), store.Declared{Declaration: []byte(`{"declaration":1,` + + `"resources":[{"id":"zsh.rc","type":"file","module":"zsh","path":"/etc/zshrc","content":"x"}]}`)}); err != nil { + t.Fatal(err) + } + if d, err := keptDeclaration(state, nil); err != nil || len(d.Resources) != 1 { + t.Fatalf("before the read is retired: %+v, %v", d, err) + } + placed := delivery.Placement{Node: "n1", Modules: []delivery.PlacedModule{{Module: "zsh"}, {Module: "lemurs"}}} + for _, row := range []struct { + name string + has map[string]bool + want bool + }{ + {"every placed module has a have", map[string]bool{"zsh": true, "lemurs": true}, true}, + {"one has none yet", map[string]bool{"zsh": true}, false}, + } { + if got, why := allHave(placed, func(m string) (bool, error) { return row.has[m], nil }); got != row.want { + t.Errorf("%s: %v (%s)", row.name, got, why) + } + } + if got, _ := allHave(delivery.Placement{Node: "n1"}, nil); got { + t.Error("a node with nothing placed retires the read") + } + keptRetired.Store(true) + defer keptRetired.Store(false) + if _, err := keptDeclaration(state, nil); !errors.Is(err, errKeptRetired) { + t.Errorf("after the read is retired: %v", err) + } + if by, filled := keptOnce(options{state: state}, nil)(); by != nil || filled != nil { + t.Errorf("after the read is retired the kept declaration was split: %v %v", by, filled) + } } // 1945/AC17 (unit): a resource the node's own composition shares with a running module's composition is declared once, diff --git a/internal/link/have_test.go b/internal/link/have_test.go index 4280811..86f5a95 100644 --- a/internal/link/have_test.go +++ b/internal/link/have_test.go @@ -274,7 +274,8 @@ func TestTheReportIsSentFirstAndTheNodeKeyCarriesTheNodesFacts(t *testing.T) { // 1126 (ADR 0555 §3): with its keys, the node key carries the node's record: the profile's present capabilities and // networks, the outward links, the health's contract and the public keys; a report that changes none of it writes // nothing, a changed fact writes the key once, and a report without a profile or health leaves them standing. - keys := RecordKeys{TunnelKey: "tk", SealingKey: "sk", ServingKey: "pk", Certificate: "cert"} + keys := RecordKeys{TunnelKey: "tk", SealingKey: "sk", ServingKey: "pk", Certificate: "cert", + Trusts: []string{"controller key sha256:aa", "vault key sha256:bb"}} // and what it trusts (L3 s6, ADR 0550 §2) q.Keys = func() RecordKeys { return keys } rep.Profile = map[string]any{"capabilities": []any{map[string]any{"name": "systemd", "present": true}, map[string]any{"name": "docker", "present": false}, map[string]any{"name": "apt", "present": true}}, @@ -286,7 +287,7 @@ func TestTheReportIsSentFirstAndTheNodeKeyCarriesTheNodesFacts(t *testing.T) { t.Fatalf("the node key holds no record: %+v, %v", got, err) } wantRecord := delivery.NodeRecord{V: delivery.Version, Node: "n1", TunnelKey: "tk", SealingKey: "sk", ServingKey: "pk", - Certificate: "cert", Capabilities: []string{"apt", "systemd"}, OutwardLinks: []string{"eth0"}, NetworksSaid: true, + Certificate: "cert", Trusts: keys.Trusts, Capabilities: []string{"apt", "systemd"}, OutwardLinks: []string{"eth0"}, NetworksSaid: true, Networks: []catalogue.AttachedNetwork{{Link: "eth0", Network: "192.0.2.0/24"}}, Contract: NodeKeyContract} if !delivery.SameNodeRecord(*got.Record, wantRecord) { t.Errorf("record = %+v, want %+v", *got.Record, wantRecord) diff --git a/internal/link/record.go b/internal/link/record.go index 1dc28a6..f176bab 100644 --- a/internal/link/record.go +++ b/internal/link/record.go @@ -23,6 +23,8 @@ type RecordKeys struct { Certificate string // Authority is the mesh's own certificate as this machine last applied it (public). Authority string + // Trusts is what this machine trusts a mesh signature from now (delivery.Trusts, ADR 0550 §2). + Trusts []string } // recordOf is the node's record from one facts report and its keys; nil for a report that carries no profile (one that @@ -45,7 +47,7 @@ func recordOf(node string, r Report, keys RecordKeys) *delivery.NodeRecord { return nil } rec := &delivery.NodeRecord{V: delivery.Version, Node: node, TunnelKey: keys.TunnelKey, SealingKey: keys.SealingKey, - ServingKey: keys.ServingKey, Certificate: keys.Certificate, Authority: keys.Authority} + ServingKey: keys.ServingKey, Certificate: keys.Certificate, Authority: keys.Authority, Trusts: keys.Trusts} for _, c := range p.Capabilities { if c.Present && c.Name != "" { rec.Capabilities = append(rec.Capabilities, c.Name) diff --git a/internal/selfcompose/selfcompose.go b/internal/selfcompose/selfcompose.go index f3167c7..015698f 100644 --- a/internal/selfcompose/selfcompose.go +++ b/internal/selfcompose/selfcompose.go @@ -12,7 +12,9 @@ package selfcompose import ( "context" + "crypto/ed25519" "crypto/sha256" + "encoding/base64" "encoding/hex" "encoding/json" "errors" @@ -27,6 +29,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/compose" "git.novox.be/novox/mesh-sdk/go/compose/frombus" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/overlay" "github.com/nats-io/nats.go" @@ -74,6 +77,13 @@ type Env struct { // Fetch reads an archive at a URL in the artifact store: a moved module's templates archive, whose texts its built // manifest carries (issue 1126). Read once per digest. Nil reads none, and a moved module does not compose. Fetch func(ctx context.Context, url string) ([]byte, error) + // VaultAuthority is the vault's authority as this node read it from the vault's `signers` (ADR 0550 §2), composed + // beside the controller's as the mesh's authority (frombus.SignersReader); "" or nil composes the controller's alone. + VaultAuthority func() string + // Certify asks the vault's `sign` for this node's own certificate, keeps it and writes the node's record again (ADR + // 0550 §2): asked only when a module placed here asks for a certificate and the record holds none from an authority + // the node trusts. Nil asks nothing, and such a module stands in, as before. + Certify func(ctx context.Context) error } // VaultSecretChanged is the vault's event that a secret changed: a name and its generation, never a value. @@ -98,6 +108,8 @@ type Composer struct { // sealedOwn is each own secret's sealed value as first handed, by module/name and generation: a value sealed again // is other bytes, so the one first had is kept while the generation stands, and the composition stays the same. sealedOwn map[string]ownSealed + // certified is the serving key a certificate was last asked for successfully, so it is asked once per key. + certified string said string refusal string // doing is what the composer waits on now (waiting), for a running answer to say (issue 1126, T2); its own lock, so @@ -497,6 +509,8 @@ func (c *Composer) compose(ctx context.Context) []agent.Update { if c.env.Running != nil { running = c.env.Running() } + // Its own certificate, asked of the vault when a module here asks for one and the record holds none it trusts. + c.certifyIfWanted(ctx, r, own) scope := frombus.Scope{Node: c.env.Node, KeptAt: kept, InOwn: inOwn, Running: running} missed := c.missed.Load() compositions, _, held, err := frombus.Compose(ctx, r, c.env.Mesh, scope, fps) @@ -731,6 +745,69 @@ func (c *Composer) Policy(ctx context.Context, module string) string { // reader is the composer's view of the bus as frombus reads it. type reader struct{ c *Composer } +// VaultAuthority is the vault's authority the node follows (frombus.SignersReader). +func (r reader) VaultAuthority(context.Context) string { + if r.c.env.VaultAuthority == nil { + return "" + } + return r.c.env.VaultAuthority() +} + +var _ frombus.SignersReader = reader{} + +// certificateWanted says whether a module the placement p puts on this node asks for a certificate for the machine's +// internal name (its manifest's `certificate`), read from the indexer's manifests; a manifest that cannot be read is +// taken as asking none, and the composition says what it misses. +func (c *Composer) certificateWanted(ctx context.Context, r reader, p delivery.Placement) bool { + for _, m := range p.Modules { + c.mu.Lock() + digest := c.modules[m.Module].Manifest + c.mu.Unlock() + if digest == "" { + continue + } + raw, err := r.Manifest(ctx, digest) + if err != nil { + continue + } + if parsed, err := catalogue.ParseManifestForInputs(raw); err == nil && parsed.Certificate != nil { + return true + } + } + return false +} + +// certifyIfWanted asks Certify once per serving key when a module placed here asks for a certificate and this node's +// record holds none for its internal name and serving key from an authority it trusts (the one it holds, or the vault's). +func (c *Composer) certifyIfWanted(ctx context.Context, r reader, p delivery.Placement) { + if c.env.Certify == nil || !c.certificateWanted(ctx, r, p) { + return + } + c.mu.Lock() + rec, ok := c.records[c.env.Node] + asked := c.certified + c.mu.Unlock() + if !ok || rec.ServingKey == "" || asked == rec.ServingKey { + return + } + key, err := base64.StdEncoding.DecodeString(rec.ServingKey) + if err != nil || len(key) != ed25519.PublicKeySize { + return + } + trusted := []string{rec.Authority, r.VaultAuthority(ctx)} + if rec.Certificate != "" && delivery.CheckNodeCertificate(rec.Certificate, trusted, overlay.InternalName(c.env.Node), + ed25519.PublicKey(key), time.Now()) == nil { + return + } + if err := c.env.Certify(ctx); err != nil { + c.sayOnce(fmt.Sprintf("a module here asks for a certificate, and the vault's sign gave none: %v", err)) + return + } + c.mu.Lock() + c.certified = rec.ServingKey + c.mu.Unlock() +} + func (r reader) Placements(context.Context) (map[string]delivery.Placement, error) { r.c.mu.Lock() defer r.c.mu.Unlock() diff --git a/internal/selfcompose/selfcompose_test.go b/internal/selfcompose/selfcompose_test.go index f588f0c..40d41f0 100644 --- a/internal/selfcompose/selfcompose_test.go +++ b/internal/selfcompose/selfcompose_test.go @@ -3,6 +3,7 @@ package selfcompose import ( "context" "crypto/sha256" + "encoding/base64" "encoding/hex" "encoding/json" "fmt" @@ -1219,6 +1220,32 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { } } b.mu.Unlock() + + // L3 s7 (ADR 0550 §2): the node asks the vault for its own certificate only when a module placed here asks for one + // and its record holds none from an authority it trusts; once per serving key. + certBus := newFakeBus() + certAsks := 0 + certEnv := certBus.env() + certEnv.Certify = func(context.Context) error { certAsks++; return nil } + certEnv.VaultAuthority = func() string { return "" } + cc := New(certEnv) + withCert := `{"module":"web","version":"1","certificate":{"into":"/etc/web/tls.crt"}}` + cdigest := delivery.ManifestDigest([]byte(withCert)) + cc.manifests[digest] = []byte(lemurs) + cc.manifests[cdigest] = []byte(withCert) + cc.modules = map[string]delivery.Module{"lemurs": {Manifest: digest}, "web": {Manifest: cdigest}} + cc.records["n1"] = delivery.NodeRecord{Node: "n1", ServingKey: base64.StdEncoding.EncodeToString(make([]byte, 32))} + onlyLemurs := delivery.Placement{Node: "n1", Modules: []delivery.PlacedModule{{Module: "lemurs"}}} + cc.certifyIfWanted(context.Background(), reader{c: cc}, onlyLemurs) + if certAsks != 0 { + t.Errorf("no module asks for a certificate, and the vault was asked %d times", certAsks) + } + withWeb := delivery.Placement{Node: "n1", Modules: []delivery.PlacedModule{{Module: "lemurs"}, {Module: "web"}}} + cc.certifyIfWanted(context.Background(), reader{c: cc}, withWeb) + cc.certifyIfWanted(context.Background(), reader{c: cc}, withWeb) + if certAsks != 1 { + t.Errorf("a module asks for a certificate: the vault was asked %d times, want once", certAsks) + } } // patternBus is a bus whose every watch gets the updates of its bucket matching its pattern, as the bus's own watches