Bring in the adoption handing; the bus node writes nothing before it heard the switch, retries the key's watch and keeps the follow last heard, and an apply never writes the controller's list while the grants part's is followed (review of the switch, issue 1945)

Merges fix/1945-adopt-hashes 1a190090. A new follower (one per link) waits for the grants part's key to catch up before its first write, so a restart after take-over never writes the controller's list first. A refused or ended watch is tried again on link.Backoff, the have says why it is not watched, and the key last heard, kept beside the node's state, holds meanwhile. While the grants part's list is followed, the bus module's composition is applied with the list on the machine in place of the controller's. The grants list is never handed to the vault. Pins mesh-sdk c2a410bb.
2026-10-26 12:30:07 +01:00
parent 61cec66ba8 1a190090ab
commit 06f70e11ee
9 changed files with 544 additions and 48 deletions
@@ -1153,7 +1153,8 @@ func runLink(ctx context.Context, opts options) error {
 		return identity.LoadSealingKey(identity.SealingKeyPath(opts.state))
 	}, func(verb string, args any) (json.RawMessage, error) {
 		return queue.AskVault(ctx, verb, args)
-	}, say, nodeKeyRetryFrom, nodeKeyRetryUpTo)
+	}, say, nodeKeyRetryFrom, nodeKeyRetryUpTo, &keyHearing{heard: vaultKey.heard,
+		held: func() bool { return vaultKey.get() != "" }, look: nodeKeyLook, every: nodeKeyReask})
 	// And whether what it runs stays up, looked at on its own clock and said when it changes (novox/hq
 	// ADR 0240) — between applies, which is when a container crash-loops.
 	if judging != nil {
@@ -412,6 +412,8 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
 				}
 			})
 		})
+		// A vault key heard (or heard again changed) looks at the list again, so its hashes are handed (ADR 0547 §9).
+		vaultKey.following(users.Again)
 		go func() {
 			if err := users.Run(linked); err != nil && linked.Err() == nil {
 				say("the bus's user list is not followed on this link: " + err.Error())
@@ -447,7 +449,9 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
 			Backoff: link.Backoff,
 			Apply: func(ctx context.Context, c delivery.Composed, others []delivery.Composed) agent.Result {
 				actx, done := applyContext(session.WithLink(ctx))
-				res := applyComposition(actx, opts, c, others, sched, say)
+				// While the grants part's list is followed, the controller's list in the bus module's composition is not
+				// written: the apply keeps the list on the machine (ADR 0547 clause 8, never both).
+				res := applyComposition(actx, opts, users.Hold(c), others, sched, say)
 				done()
 				// An apply of the bus module may write an older list than the newest composition holds: looked at again.
 				if module, _ := users.State(); module != "" && module == c.Module {
@@ -2,7 +2,10 @@ package main

 import (
 	"context"
+	"encoding/json"
 	"fmt"
+	"strings"
+	"sync"
 	"time"

 	"github.com/novox/mesh-host/internal/identity"
@@ -22,12 +25,77 @@ const (
 	nodeKeyLook = 30 * time.Second
 )

+// heardKey is the vault's own public key as its answer to node-key gave it (ADR 0547 clause 9, issue 1945): the key the
+// bus node hands the loaded list's hashes sealed to. One for the process, as the key stated is; kick is the follower of
+// the user list on the link open now, told when the key is first heard or changes.
+type heardKey struct {
+	mu   sync.Mutex
+	pem  string
+	kick func()
+}
+
+// vaultKey is the vault's key this engine heard.
+var vaultKey = &heardKey{}
+
+func (h *heardKey) get() string {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	return h.pem
+}
+
+// heard keeps the key a node-key answer gave, and tells the follower when it is a first or changed one. An answer
+// with no key (a vault before this one) changes nothing.
+func (h *heardKey) heard(answer json.RawMessage) {
+	var a struct {
+		VaultKey string `json:"vault_key"`
+	}
+	if json.Unmarshal(answer, &a) != nil || !strings.Contains(a.VaultKey, "PUBLIC KEY") {
+		return
+	}
+	h.mu.Lock()
+	changed := a.VaultKey != h.pem
+	h.pem = a.VaultKey
+	kick := h.kick
+	h.mu.Unlock()
+	if changed && kick != nil {
+		kick()
+	}
+}
+
+// following names the follower told when the key changes, and tells it at once when a key is known.
+func (h *heardKey) following(kick func()) {
+	h.mu.Lock()
+	h.kick = kick
+	known := h.pem != ""
+	h.mu.Unlock()
+	if known && kick != nil {
+		kick()
+	}
+}
+
+// nodeKeyReask is how often the key is stated again though it did not change, so a changed vault key is heard: the
+// vault takes the same key again and changes nothing (ADR 0547 clause 9).
+const nodeKeyReask = 10 * time.Minute
+
+// keyHearing is what the keeper does with the vault's own key in node-key's answers (ADR 0547 clause 9): heard takes
+// it from an answer; held says whether one is held, and while none is the key is stated again at every look, so a vault
+// delivered after this engine is heard within a look; every is how often it is stated again all the same, so a changed
+// vault key is heard.
+type keyHearing struct {
+	heard func(json.RawMessage)
+	held  func() bool
+	// look is how often it is stated again while no vault key is held; every how often once one is.
+	look, every time.Duration
+}
+
 // keepNodeKeyStated states the sealing key load answers to the vault at start, retrying with a growing wait from
 // retryFrom to retryUpTo while it does not answer, and states it again whenever the key changes. It says one line
-// when a key is stated and one when a try fails for another reason than the last, and returns when ctx ends.
+// when a key is stated and one when a try fails for another reason than the last, and returns when ctx ends. With
+// hearing, each answer goes to it, and the key is stated again while no vault key is held and at its interval.
 func keepNodeKeyStated(ctx context.Context, load func() (identity.SealingKey, error), ask vaultAsk, say func(string),
-	retryFrom, retryUpTo time.Duration) {
+	retryFrom, retryUpTo time.Duration, hearing *keyHearing) {
 	stated, lastFailure := "", ""
+	var askedAt time.Time
 	wait := retryFrom
 	fail := func(why string) {
 		if why != lastFailure {
@@ -40,17 +108,29 @@ func keepNodeKeyStated(ctx context.Context, load func() (identity.SealingKey, er
 	for {
 		delay := look
 		key, err := load()
+		again := false
+		if hearing != nil && stated != "" {
+			since := time.Since(askedAt)
+			held := hearing.held == nil || hearing.held()
+			again = (!held && since >= hearing.look) || (held && hearing.every > 0 && since >= hearing.every)
+		}
 		switch {
 		case err != nil:
 			fail(err.Error())
 			delay = wait
-		case key.Public != stated:
-			if _, err := ask("node-key", map[string]string{"key": key.Public}); err != nil {
+		case key.Public != stated || again:
+			if answer, err := ask("node-key", map[string]string{"key": key.Public}); err != nil {
 				fail(err.Error())
 				delay = wait
 			} else {
+				askedAt = time.Now()
+				if hearing != nil && hearing.heard != nil {
+					hearing.heard(answer)
+				}
+				if key.Public != stated {
+					say(fmt.Sprintf("mesh-host: this machine's sealing key %s is stated to the vault", key.Public))
+				}
 				stated, lastFailure, wait = key.Public, "", retryFrom
-				say(fmt.Sprintf("mesh-host: this machine's sealing key %s is stated to the vault", key.Public))
 			}
 		}
 		select {
@@ -19,6 +19,10 @@ type keyVault struct {
 	stated []string
 	fail   int
 	verbs  []string
+	// keyless is how many answers carry no vault key (a vault before ADR 0547 clause 9's, or not yet started); key the
+	// vault's key the others carry.
+	keyless int
+	key     string
 }

 func (v *keyVault) ask(verb string, args any) (json.RawMessage, error) {
@@ -35,7 +39,12 @@ func (v *keyVault) ask(verb string, args any) (json.RawMessage, error) {
 	}
 	_ = json.Unmarshal(raw, &a)
 	v.stated = append(v.stated, a.Key)
-	return json.RawMessage(`{"recorded":true}`), nil
+	if v.key == "" || v.keyless > 0 {
+		v.keyless--
+		return json.RawMessage(`{"recorded":true}`), nil
+	}
+	answer, _ := json.Marshal(map[string]any{"recorded": true, "vault_key": v.key})
+	return answer, nil
 }

 func (v *keyVault) said() []string {
@@ -71,9 +80,14 @@ func TestTheEngineStatesItsSealingKeyToTheVaultAtStartRetriesAndRestatesAChange(
 	say := func(l string) { lmu.Lock(); lines = append(lines, l); lmu.Unlock() }
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	go keepNodeKeyStated(ctx, load, vault.ask, say, time.Millisecond, 5*time.Millisecond)
+	var hmu sync.Mutex
+	var answers []string
+	heard := func(a json.RawMessage) { hmu.Lock(); answers = append(answers, string(a)); hmu.Unlock() }
+	go keepNodeKeyStated(ctx, load, vault.ask, say, time.Millisecond, 5*time.Millisecond, &keyHearing{heard: heard})

 	waitFor(t, "the first key stated after two refusals", func() bool { return len(vault.said()) == 1 })
+	// Its answer is handed on, so the vault's own key in it is heard (ADR 0547 clause 9); refusals hand nothing.
+	waitFor(t, "the answer heard", func() bool { hmu.Lock(); defer hmu.Unlock(); return len(answers) == 1 })
 	if vault.said()[0] != first.Public {
 		t.Errorf("stated %q, want the engine's own key %q", vault.said()[0], first.Public)
 	}
@@ -102,11 +116,75 @@ func TestTheEngineStatesItsSealingKeyToTheVaultAtStartRetriesAndRestatesAChange(
 	if len(lines) == 0 {
 		t.Error("the engine said nothing")
 	}
+	asksAgainUntilItHearsTheVaultsKey(t)
 }

-// The statement a host makes names this contract (issue 786), one above the vault's box.
+// ADR 0547 clause 9: the vault's key is heard though the engine's own key never changes. While none is held the key
+// is stated again at each look, so a vault delivered after the engine (or answering no key yet) is heard; once one is
+// held, again only at the slow interval, so a changed vault key is heard; never more.
+func asksAgainUntilItHearsTheVaultsKey(t *testing.T) {
+	sealing, _ := identity.GenerateSealingKey()
+	load := func() (identity.SealingKey, error) { return sealing, nil }
+	pem := func(c string) string { return "-----BEGIN PUBLIC KEY-----\n" + c + "\n-----END PUBLIC KEY-----\n" }
+	for _, c := range []struct {
+		name        string
+		keyless     int
+		every       time.Duration
+		change      bool
+		wantAtLeast int
+		wantAtMost  int
+	}{
+		{name: "a vault with no key yet is asked again until it answers one, then not", keyless: 3, wantAtLeast: 4, wantAtMost: 4},
+		{name: "a held key is asked again at the slow interval, and a changed one heard", every: 20 * time.Millisecond,
+			change: true, wantAtLeast: 3, wantAtMost: 1 << 20},
+	} {
+		t.Run(c.name, func(t *testing.T) {
+			vault := &keyVault{keyless: c.keyless, key: pem("AAAA")}
+			h := &heardKey{}
+			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
+			go keepNodeKeyStated(ctx, load, vault.ask, func(string) {}, time.Millisecond, 5*time.Millisecond,
+				&keyHearing{heard: h.heard, held: func() bool { return h.get() != "" }, look: time.Millisecond, every: c.every})
+			waitFor(t, "the vault's key heard", func() bool { return h.get() == pem("AAAA") })
+			if c.change {
+				vault.mu.Lock()
+				vault.key = pem("BBBB")
+				vault.mu.Unlock()
+				waitFor(t, "the changed vault key heard", func() bool { return h.get() == pem("BBBB") })
+			}
+			time.Sleep(60 * time.Millisecond)
+			n := len(vault.said())
+			if n < c.wantAtLeast || n > c.wantAtMost {
+				t.Errorf("node-key asked %d times, want %d to %d", n, c.wantAtLeast, c.wantAtMost)
+			}
+		})
+	}
+}
+
+// The statement a host makes names this contract (issue 786), one above the vault's box. And the vault's key its
+// answer gives is kept, told to the follower once when first heard and again only when changed; an answer without one
+// (an older vault) or with no public key changes nothing (ADR 0547 clause 9).
 func TestTheNodeKeyContractIsThirteen(t *testing.T) {
 	if link.NodeKeyContract != 13 {
 		t.Errorf("NodeKeyContract %d", link.NodeKeyContract)
 	}
+	h := &heardKey{}
+	kicks := 0
+	h.following(func() { kicks++ })
+	h.heard(json.RawMessage(`{"recorded":true}`))
+	h.heard(json.RawMessage(`{"recorded":true,"vault_key":"not a key"}`))
+	if h.get() != "" || kicks != 0 {
+		t.Fatalf("an answer with no key was kept: %q, %d kicks", h.get(), kicks)
+	}
+	pem := "-----BEGIN PUBLIC KEY-----\nAAAA\n-----END PUBLIC KEY-----\n"
+	answer, _ := json.Marshal(map[string]any{"recorded": true, "vault_key": pem})
+	h.heard(answer)
+	h.heard(answer)
+	if h.get() != pem || kicks != 1 {
+		t.Fatalf("key %q, %d kicks, want the key and one", h.get(), kicks)
+	}
+	h.following(func() { kicks += 10 })
+	if kicks != 11 {
+		t.Fatalf("a follower named after the key was heard was not told: %d", kicks)
+	}
 }
@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"path/filepath"

 	"git.novox.be/novox/mesh-sdk/go/declaration"
 	"git.novox.be/novox/mesh-sdk/go/delivery"
@@ -69,7 +70,15 @@ func userListFollower(opts options, session *link.Session, composedBucket string
 			}
 			return pem
 		},
-		Say: announceOr(say),
+		// The grants part's key as last heard, kept beside the node's state, followed while it cannot be watched; its
+		// watch tried again as a passing failure of the bus is (ADR 0547 clause 8).
+		Kept:    func() ([]byte, uint64) { return keptGrantsKey(opts) },
+		Keep:    func(value []byte, rev uint64) { keepGrantsKey(opts, value, rev, say) },
+		Backoff: link.Backoff,
+		// The vault's key from its answer to node-key: the loaded list's hashes are handed sealed to it (ADR 0547
+		// clause 9).
+		VaultKey: vaultKey.get,
+		Say:      announceOr(say),
 	})
 }

@@ -152,3 +161,44 @@ func writeUserListAlone(ctx context.Context, opts options, raw []byte, body []by
 	}
 	return digest, writeErr == nil && out.Action != "unchanged", writeErr
 }
+
+// grantsKeyPath is where the grants part's user-list key last heard is kept, beside the node's state.
+func grantsKeyPath(opts options) string {
+	return filepath.Join(filepath.Dir(opts.state), "grants-user-list.json")
+}
+
+type keptKey struct {
+	Revision uint64          `json:"revision"`
+	Value    json.RawMessage `json:"value"`
+}
+
+func keptGrantsKey(opts options) ([]byte, uint64) {
+	raw, err := os.ReadFile(grantsKeyPath(opts))
+	if err != nil {
+		return nil, 0
+	}
+	var k keptKey
+	if json.Unmarshal(raw, &k) != nil || len(k.Value) == 0 {
+		return nil, 0
+	}
+	return k.Value, k.Revision
+}
+
+func keepGrantsKey(opts options, value []byte, rev uint64, say link.Announce) {
+	path := grantsKeyPath(opts)
+	if value == nil {
+		if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
+			announceOr(say)("the grants part's user-list key kept on this machine could not be taken away: " + err.Error())
+		}
+		return
+	}
+	raw, _ := json.Marshal(keptKey{Revision: rev, Value: value})
+	tmp := path + ".new"
+	err := os.WriteFile(tmp, raw, 0o600)
+	if err == nil {
+		err = os.Rename(tmp, path)
+	}
+	if err != nil {
+		announceOr(say)("the grants part's user-list key could not be kept on this machine: " + err.Error())
+	}
+}