diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index 06ebfd9..ade268e 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -1153,7 +1153,8 @@ func runLink(ctx context.Context, opts options) error { return identity.LoadSealingKey(identity.SealingKeyPath(opts.state)) }, func(verb string, args any) (json.RawMessage, error) { return queue.AskVault(ctx, verb, args) - }, say, nodeKeyRetryFrom, nodeKeyRetryUpTo) + }, say, nodeKeyRetryFrom, nodeKeyRetryUpTo, &keyHearing{heard: vaultKey.heard, + held: func() bool { return vaultKey.get() != "" }, look: nodeKeyLook, every: nodeKeyReask}) // And whether what it runs stays up, looked at on its own clock and said when it changes (novox/hq // ADR 0240) — between applies, which is when a container crash-loops. if judging != nil { diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index 8c47ac4..895403e 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -412,6 +412,8 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess } }) }) + // A vault key heard (or heard again changed) looks at the list again, so its hashes are handed (ADR 0547 §9). + vaultKey.following(users.Again) go func() { if err := users.Run(linked); err != nil && linked.Err() == nil { say("the bus's user list is not followed on this link: " + err.Error()) @@ -447,7 +449,9 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess Backoff: link.Backoff, Apply: func(ctx context.Context, c delivery.Composed, others []delivery.Composed) agent.Result { actx, done := applyContext(session.WithLink(ctx)) - res := applyComposition(actx, opts, c, others, sched, say) + // While the grants part's list is followed, the controller's list in the bus module's composition is not + // written: the apply keeps the list on the machine (ADR 0547 clause 8, never both). + res := applyComposition(actx, opts, users.Hold(c), others, sched, say) done() // An apply of the bus module may write an older list than the newest composition holds: looked at again. if module, _ := users.State(); module != "" && module == c.Module { diff --git a/cmd/mesh-host/nodekey.go b/cmd/mesh-host/nodekey.go index f00c2f8..5881baf 100644 --- a/cmd/mesh-host/nodekey.go +++ b/cmd/mesh-host/nodekey.go @@ -2,7 +2,10 @@ package main import ( "context" + "encoding/json" "fmt" + "strings" + "sync" "time" "github.com/novox/mesh-host/internal/identity" @@ -22,12 +25,77 @@ const ( nodeKeyLook = 30 * time.Second ) +// heardKey is the vault's own public key as its answer to node-key gave it (ADR 0547 clause 9, issue 1945): the key the +// bus node hands the loaded list's hashes sealed to. One for the process, as the key stated is; kick is the follower of +// the user list on the link open now, told when the key is first heard or changes. +type heardKey struct { + mu sync.Mutex + pem string + kick func() +} + +// vaultKey is the vault's key this engine heard. +var vaultKey = &heardKey{} + +func (h *heardKey) get() string { + h.mu.Lock() + defer h.mu.Unlock() + return h.pem +} + +// heard keeps the key a node-key answer gave, and tells the follower when it is a first or changed one. An answer +// with no key (a vault before this one) changes nothing. +func (h *heardKey) heard(answer json.RawMessage) { + var a struct { + VaultKey string `json:"vault_key"` + } + if json.Unmarshal(answer, &a) != nil || !strings.Contains(a.VaultKey, "PUBLIC KEY") { + return + } + h.mu.Lock() + changed := a.VaultKey != h.pem + h.pem = a.VaultKey + kick := h.kick + h.mu.Unlock() + if changed && kick != nil { + kick() + } +} + +// following names the follower told when the key changes, and tells it at once when a key is known. +func (h *heardKey) following(kick func()) { + h.mu.Lock() + h.kick = kick + known := h.pem != "" + h.mu.Unlock() + if known && kick != nil { + kick() + } +} + +// nodeKeyReask is how often the key is stated again though it did not change, so a changed vault key is heard: the +// vault takes the same key again and changes nothing (ADR 0547 clause 9). +const nodeKeyReask = 10 * time.Minute + +// keyHearing is what the keeper does with the vault's own key in node-key's answers (ADR 0547 clause 9): heard takes +// it from an answer; held says whether one is held, and while none is the key is stated again at every look, so a vault +// delivered after this engine is heard within a look; every is how often it is stated again all the same, so a changed +// vault key is heard. +type keyHearing struct { + heard func(json.RawMessage) + held func() bool + // look is how often it is stated again while no vault key is held; every how often once one is. + look, every time.Duration +} + // keepNodeKeyStated states the sealing key load answers to the vault at start, retrying with a growing wait from // retryFrom to retryUpTo while it does not answer, and states it again whenever the key changes. It says one line -// when a key is stated and one when a try fails for another reason than the last, and returns when ctx ends. +// when a key is stated and one when a try fails for another reason than the last, and returns when ctx ends. With +// hearing, each answer goes to it, and the key is stated again while no vault key is held and at its interval. func keepNodeKeyStated(ctx context.Context, load func() (identity.SealingKey, error), ask vaultAsk, say func(string), - retryFrom, retryUpTo time.Duration) { + retryFrom, retryUpTo time.Duration, hearing *keyHearing) { stated, lastFailure := "", "" + var askedAt time.Time wait := retryFrom fail := func(why string) { if why != lastFailure { @@ -40,17 +108,29 @@ func keepNodeKeyStated(ctx context.Context, load func() (identity.SealingKey, er for { delay := look key, err := load() + again := false + if hearing != nil && stated != "" { + since := time.Since(askedAt) + held := hearing.held == nil || hearing.held() + again = (!held && since >= hearing.look) || (held && hearing.every > 0 && since >= hearing.every) + } switch { case err != nil: fail(err.Error()) delay = wait - case key.Public != stated: - if _, err := ask("node-key", map[string]string{"key": key.Public}); err != nil { + case key.Public != stated || again: + if answer, err := ask("node-key", map[string]string{"key": key.Public}); err != nil { fail(err.Error()) delay = wait } else { + askedAt = time.Now() + if hearing != nil && hearing.heard != nil { + hearing.heard(answer) + } + if key.Public != stated { + say(fmt.Sprintf("mesh-host: this machine's sealing key %s is stated to the vault", key.Public)) + } stated, lastFailure, wait = key.Public, "", retryFrom - say(fmt.Sprintf("mesh-host: this machine's sealing key %s is stated to the vault", key.Public)) } } select { diff --git a/cmd/mesh-host/nodekey_test.go b/cmd/mesh-host/nodekey_test.go index 80f85ce..0bae9f5 100644 --- a/cmd/mesh-host/nodekey_test.go +++ b/cmd/mesh-host/nodekey_test.go @@ -19,6 +19,10 @@ type keyVault struct { stated []string fail int verbs []string + // keyless is how many answers carry no vault key (a vault before ADR 0547 clause 9's, or not yet started); key the + // vault's key the others carry. + keyless int + key string } func (v *keyVault) ask(verb string, args any) (json.RawMessage, error) { @@ -35,7 +39,12 @@ func (v *keyVault) ask(verb string, args any) (json.RawMessage, error) { } _ = json.Unmarshal(raw, &a) v.stated = append(v.stated, a.Key) - return json.RawMessage(`{"recorded":true}`), nil + if v.key == "" || v.keyless > 0 { + v.keyless-- + return json.RawMessage(`{"recorded":true}`), nil + } + answer, _ := json.Marshal(map[string]any{"recorded": true, "vault_key": v.key}) + return answer, nil } func (v *keyVault) said() []string { @@ -71,9 +80,14 @@ func TestTheEngineStatesItsSealingKeyToTheVaultAtStartRetriesAndRestatesAChange( say := func(l string) { lmu.Lock(); lines = append(lines, l); lmu.Unlock() } ctx, cancel := context.WithCancel(context.Background()) defer cancel() - go keepNodeKeyStated(ctx, load, vault.ask, say, time.Millisecond, 5*time.Millisecond) + var hmu sync.Mutex + var answers []string + heard := func(a json.RawMessage) { hmu.Lock(); answers = append(answers, string(a)); hmu.Unlock() } + go keepNodeKeyStated(ctx, load, vault.ask, say, time.Millisecond, 5*time.Millisecond, &keyHearing{heard: heard}) waitFor(t, "the first key stated after two refusals", func() bool { return len(vault.said()) == 1 }) + // Its answer is handed on, so the vault's own key in it is heard (ADR 0547 clause 9); refusals hand nothing. + waitFor(t, "the answer heard", func() bool { hmu.Lock(); defer hmu.Unlock(); return len(answers) == 1 }) if vault.said()[0] != first.Public { t.Errorf("stated %q, want the engine's own key %q", vault.said()[0], first.Public) } @@ -102,11 +116,75 @@ func TestTheEngineStatesItsSealingKeyToTheVaultAtStartRetriesAndRestatesAChange( if len(lines) == 0 { t.Error("the engine said nothing") } + asksAgainUntilItHearsTheVaultsKey(t) } -// The statement a host makes names this contract (issue 786), one above the vault's box. +// ADR 0547 clause 9: the vault's key is heard though the engine's own key never changes. While none is held the key +// is stated again at each look, so a vault delivered after the engine (or answering no key yet) is heard; once one is +// held, again only at the slow interval, so a changed vault key is heard; never more. +func asksAgainUntilItHearsTheVaultsKey(t *testing.T) { + sealing, _ := identity.GenerateSealingKey() + load := func() (identity.SealingKey, error) { return sealing, nil } + pem := func(c string) string { return "-----BEGIN PUBLIC KEY-----\n" + c + "\n-----END PUBLIC KEY-----\n" } + for _, c := range []struct { + name string + keyless int + every time.Duration + change bool + wantAtLeast int + wantAtMost int + }{ + {name: "a vault with no key yet is asked again until it answers one, then not", keyless: 3, wantAtLeast: 4, wantAtMost: 4}, + {name: "a held key is asked again at the slow interval, and a changed one heard", every: 20 * time.Millisecond, + change: true, wantAtLeast: 3, wantAtMost: 1 << 20}, + } { + t.Run(c.name, func(t *testing.T) { + vault := &keyVault{keyless: c.keyless, key: pem("AAAA")} + h := &heardKey{} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go keepNodeKeyStated(ctx, load, vault.ask, func(string) {}, time.Millisecond, 5*time.Millisecond, + &keyHearing{heard: h.heard, held: func() bool { return h.get() != "" }, look: time.Millisecond, every: c.every}) + waitFor(t, "the vault's key heard", func() bool { return h.get() == pem("AAAA") }) + if c.change { + vault.mu.Lock() + vault.key = pem("BBBB") + vault.mu.Unlock() + waitFor(t, "the changed vault key heard", func() bool { return h.get() == pem("BBBB") }) + } + time.Sleep(60 * time.Millisecond) + n := len(vault.said()) + if n < c.wantAtLeast || n > c.wantAtMost { + t.Errorf("node-key asked %d times, want %d to %d", n, c.wantAtLeast, c.wantAtMost) + } + }) + } +} + +// The statement a host makes names this contract (issue 786), one above the vault's box. And the vault's key its +// answer gives is kept, told to the follower once when first heard and again only when changed; an answer without one +// (an older vault) or with no public key changes nothing (ADR 0547 clause 9). func TestTheNodeKeyContractIsThirteen(t *testing.T) { if link.NodeKeyContract != 13 { t.Errorf("NodeKeyContract %d", link.NodeKeyContract) } + h := &heardKey{} + kicks := 0 + h.following(func() { kicks++ }) + h.heard(json.RawMessage(`{"recorded":true}`)) + h.heard(json.RawMessage(`{"recorded":true,"vault_key":"not a key"}`)) + if h.get() != "" || kicks != 0 { + t.Fatalf("an answer with no key was kept: %q, %d kicks", h.get(), kicks) + } + pem := "-----BEGIN PUBLIC KEY-----\nAAAA\n-----END PUBLIC KEY-----\n" + answer, _ := json.Marshal(map[string]any{"recorded": true, "vault_key": pem}) + h.heard(answer) + h.heard(answer) + if h.get() != pem || kicks != 1 { + t.Fatalf("key %q, %d kicks, want the key and one", h.get(), kicks) + } + h.following(func() { kicks += 10 }) + if kicks != 11 { + t.Fatalf("a follower named after the key was heard was not told: %d", kicks) + } } diff --git a/cmd/mesh-host/userlist.go b/cmd/mesh-host/userlist.go index a0d58fc..9d4ac17 100644 --- a/cmd/mesh-host/userlist.go +++ b/cmd/mesh-host/userlist.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "os" + "path/filepath" "git.novox.be/novox/mesh-sdk/go/declaration" "git.novox.be/novox/mesh-sdk/go/delivery" @@ -69,7 +70,15 @@ func userListFollower(opts options, session *link.Session, composedBucket string } return pem }, - Say: announceOr(say), + // The grants part's key as last heard, kept beside the node's state, followed while it cannot be watched; its + // watch tried again as a passing failure of the bus is (ADR 0547 clause 8). + Kept: func() ([]byte, uint64) { return keptGrantsKey(opts) }, + Keep: func(value []byte, rev uint64) { keepGrantsKey(opts, value, rev, say) }, + Backoff: link.Backoff, + // The vault's key from its answer to node-key: the loaded list's hashes are handed sealed to it (ADR 0547 + // clause 9). + VaultKey: vaultKey.get, + Say: announceOr(say), }) } @@ -152,3 +161,44 @@ func writeUserListAlone(ctx context.Context, opts options, raw []byte, body []by } return digest, writeErr == nil && out.Action != "unchanged", writeErr } + +// grantsKeyPath is where the grants part's user-list key last heard is kept, beside the node's state. +func grantsKeyPath(opts options) string { + return filepath.Join(filepath.Dir(opts.state), "grants-user-list.json") +} + +type keptKey struct { + Revision uint64 `json:"revision"` + Value json.RawMessage `json:"value"` +} + +func keptGrantsKey(opts options) ([]byte, uint64) { + raw, err := os.ReadFile(grantsKeyPath(opts)) + if err != nil { + return nil, 0 + } + var k keptKey + if json.Unmarshal(raw, &k) != nil || len(k.Value) == 0 { + return nil, 0 + } + return k.Value, k.Revision +} + +func keepGrantsKey(opts options, value []byte, rev uint64, say link.Announce) { + path := grantsKeyPath(opts) + if value == nil { + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + announceOr(say)("the grants part's user-list key kept on this machine could not be taken away: " + err.Error()) + } + return + } + raw, _ := json.Marshal(keptKey{Revision: rev, Value: value}) + tmp := path + ".new" + err := os.WriteFile(tmp, raw, 0o600) + if err == nil { + err = os.Rename(tmp, path) + } + if err != nil { + announceOr(say)("the grants part's user-list key could not be kept on this machine: " + err.Error()) + } +} diff --git a/go.mod b/go.mod index 46d0955..a5334da 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/novox/mesh-host go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026110029-83ee7769d8ee + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026112527-c2a410bb6376 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 golang.org/x/crypto v0.57.0 diff --git a/go.sum b/go.sum index 87bf518..976c4b4 100644 --- a/go.sum +++ b/go.sum @@ -8,6 +8,10 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026105311-175a5567b8bd h1:ef6y09cs git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026105311-175a5567b8bd/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026110029-83ee7769d8ee h1:ZdBMkF4WYJi82jb3xYi9Z2lXN/iMTn9r3SFC122ROY4= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026110029-83ee7769d8ee/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026112307-af66adc180cd h1:DV4xfPRqmh6p20D2QNN+9EGFAyvpGOZPTRsSbpT7oww= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026112307-af66adc180cd/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026112527-c2a410bb6376 h1:by2zBASLb8h5jA1IiYJWykXwLnHYpuZh/zFNa4NsbOU= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026112527-c2a410bb6376/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/internal/userlist/userlist.go b/internal/userlist/userlist.go index 7653ac2..1a9a878 100644 --- a/internal/userlist/userlist.go +++ b/internal/userlist/userlist.go @@ -16,6 +16,11 @@ // // It reads no want and writes none, moves no fingerprint, and touches nothing of the bus module but its user list. // +// **It hands the loaded list's hashes to the vault** (ADR 0547 clause 9, issue 1945): once the vault's key is known +// (VaultKey), the have of the bus module carries the bcrypt hash of every user of the list on the machine, sealed to +// that key (delivery.AccountsHanded), so the vault adopts the accounts it holds no value for. It is sealed again only +// when the list on the machine or the vault's key changes. +// // **From take-over it follows the grants part's list instead** (ADR 0547 clauses 7 and 8, issue 1945). On the node whose // compositions carry the list, it also watches the grants part's one key `user-list.sealed`. While that key says // FollowGrants, the list is the object the key names, read by its name, opened with this node's sealing key and checked @@ -74,8 +79,17 @@ type Env struct { Fetch func(ctx context.Context, name string) ([]byte, error) Open func(seal.Box) ([]byte, error) Key func() string - Now func() time.Time - Say func(string) + // Kept is the grants part's key as last heard on this machine and its revision (nil when none), and Keep keeps it: + // what the follower follows while the key cannot be watched (ADR 0547 clause 8). Backoff is the wait before the + // try-th watch again. + Kept func() ([]byte, uint64) + Keep func(value []byte, rev uint64) + Backoff func(try int) time.Duration + // VaultKey is the vault's own public key (SPKI PEM) as its answer to node-key gave it, "" while none is known: the + // key the loaded list's hashes are handed sealed to. Nil hands nothing. + VaultKey func() string + Now func() time.Time + Say func(string) } type source struct { @@ -103,6 +117,8 @@ type Follower struct { grants *delivery.GrantsUserList grantsRev uint64 following string + // unwatched is why the grants part's key is not watched now; empty while it is, or where it need not be. + unwatched string } // New is a follower for a node. @@ -113,7 +129,18 @@ func New(env Env) *Follower { if env.Say == nil { env.Say = func(string) {} } - return &Follower{env: env, kick: make(chan struct{}, 1), srcs: map[string]source{}, said: map[string]bool{}} + if env.Backoff == nil { + env.Backoff = func(int) time.Duration { return time.Minute } + } + f := &Follower{env: env, kick: make(chan struct{}, 1), srcs: map[string]source{}, said: map[string]bool{}} + if env.Grants != nil && env.Kept != nil { + if raw, rev := env.Kept(); raw != nil { + if g, err := delivery.DecodeGrantsUserList(raw); err == nil { + f.grants, f.grantsRev = &g, rev + } + } + } + return f } // Again asks the follower to look at the file again, though no composition changed: after an apply of the bus module, @@ -140,7 +167,33 @@ func (f *Follower) Run(ctx context.Context) error { return fmt.Errorf("the user list cannot be followed: %w", err) } caught := false + // The grants part's key, on the node whose compositions carry the list: the bus node alone is granted it (mesh-sdk + // buscompose.GrantsUserListFollows). Until its watch caught up nothing is written, so a new follower (one per link) + // never writes the controller's list after take-over before it heard the switch; a watch refused or ended is tried + // again on the backoff, and meanwhile the follow last heard (kept on the machine) holds. var grants <-chan agent.Update + var retry <-chan time.Time + opened, grantsCaught, tries := false, false, 0 + failed := func(why string) { + grants, tries = nil, tries+1 + retry = time.After(f.env.Backoff(tries)) + f.mu.Lock() + f.unwatched = why + f.mu.Unlock() + f.sayOnce("the grants part's user-list key is not watched, tried again: " + why) + } + open := func() { + if grants != nil || retry != nil || f.env.Grants == nil || !f.busNode() { + return + } + opened = true + ch, err := f.env.Grants.Watch(ctx, delivery.GrantsUserListKey) + if err != nil { + failed(err.Error()) + return + } + grants = ch + } for { again := false select { @@ -148,6 +201,8 @@ func (f *Follower) Run(ctx context.Context) error { return nil case <-f.kick: again = true + case <-retry: + retry = nil case u, ok := <-updates: if !ok { return fmt.Errorf("the watch of the compositions ended") @@ -159,29 +214,31 @@ func (f *Follower) Run(ctx context.Context) error { } case u, ok := <-grants: if !ok { - // Its watch ended (the link, or a grant the bus took away): heard again at the next event. - grants = nil - f.sayOnce("the grants part's user list is not followed until its key can be watched again") - continue + failed("its watch ended") + break } if u.Caught { - continue // nothing changed by the watch catching up + grantsCaught, tries = true, 0 + f.mu.Lock() + f.unwatched = "" + f.mu.Unlock() + break } f.takeGrants(u) } - if caught { - f.settle(ctx, again) - // The grants part's key is watched on the node whose compositions carry the list: the bus node alone is - // granted it (mesh-sdk buscompose.GrantsUserListFollows). - if grants == nil && f.env.Grants != nil && f.busNode() { - ch, err := f.env.Grants.Watch(ctx, delivery.GrantsUserListKey) - if err != nil { - f.sayOnce("the grants part's user list cannot be watched yet, so the controller's is followed: " + err.Error()) - } else { - grants = ch - } - } + if !caught { + continue } + open() + if opened && !grantsCaught && grants != nil { + continue // the switch is not heard yet: nothing is written before it is + } + f.mu.Lock() + if f.unwatched != f.state.Unwatched { + again = true // the have says whether the key is watched + } + f.mu.Unlock() + f.settle(ctx, again) } } @@ -200,15 +257,25 @@ func (f *Follower) takeGrants(u agent.Update) { f.grantsRev = u.Revision if u.Deleted { f.grants = nil + f.keep(nil, 0) return } g, err := delivery.DecodeGrantsUserList(u.Value) if err != nil { f.grants = nil + f.keep(nil, 0) f.sayOnceLocked("the grants part's user-list key cannot be read, so the controller's list is followed: " + err.Error()) return } f.grants = &g + f.keep(u.Value, u.Revision) +} + +// keep keeps the key last heard on the machine (Env.Keep), so a follower that cannot watch it follows what it said. +func (f *Follower) keep(value []byte, rev uint64) { + if f.env.Keep != nil { + f.env.Keep(value, rev) + } } // Following is which list the follower takes now: the grants part's while its key says so, the controller's otherwise. @@ -225,6 +292,12 @@ func (f *Follower) followingLocked() string { return delivery.FollowController } +func (f *Follower) unwatchedNow() string { + f.mu.Lock() + defer f.mu.Unlock() + return f.unwatched +} + func (f *Follower) key() string { if f.env.Key == nil { return "" @@ -308,7 +381,7 @@ func (f *Follower) settle(ctx context.Context, again bool) { f.mu.Unlock() w := Written{Source: newest.key, Revision: newest.rev, Wanted: newest.wanted, At: f.env.Now().UTC(), - Follows: delivery.FollowController, Key: f.key()} + Follows: delivery.FollowController, Key: f.key(), Unwatched: f.unwatchedNow()} var changed bool body, err := f.env.Resolve(ctx, newest.resource) if err == nil { @@ -321,14 +394,24 @@ func (f *Follower) settle(ctx context.Context, again bool) { } } } + var loaded []byte if err == nil { w.Digest, changed, err = f.env.Write(ctx, newest.module, newest.resource, body) + if err == nil { + loaded = body + } + } else if body != nil && f.env.OnDisk != nil { + // Refused: the list on the machine stays, and it is the one the bus has loaded. + loaded, _ = f.env.OnDisk(newest.resource) } if err != nil { w.Error = err.Error() } f.mu.Lock() prev := f.state + f.mu.Unlock() + w.Handed = f.hand(loaded, prev.Handed) + f.mu.Lock() f.mod, f.state = newest.module, w f.written = err == nil f.following = delivery.FollowController @@ -348,7 +431,8 @@ func (f *Follower) settle(ctx context.Context, again bool) { // reportable says a have's user list says something new: a look again that found the file as it was says nothing. func reportable(prev, w Written) bool { return prev.Source != w.Source || prev.Revision != w.Revision || prev.Digest != w.Digest || prev.Error != w.Error || - prev.Follows != w.Follows || prev.Key != w.Key + prev.Follows != w.Follows || prev.Key != w.Key || handedKey(prev.Handed) != handedKey(w.Handed) || + prev.Unwatched != w.Unwatched } // settleGrants writes the grants part's list, the object its key names, into the file the controller's composition of @@ -369,7 +453,7 @@ func (f *Follower) settleGrants(ctx context.Context, again bool) { f.mu.Unlock() w := Written{Source: delivery.GrantsUserListSource, Revision: rev, Wanted: g.Digest, At: f.env.Now().UTC(), - Follows: delivery.FollowGrants, Key: f.key()} + Follows: delivery.FollowGrants, Key: f.key(), Unwatched: f.unwatchedNow()} var err error var changed bool resource, rerr := mode0600(file.resource) @@ -413,6 +497,8 @@ func (f *Follower) settleGrants(ctx context.Context, again bool) { } f.mu.Lock() prev := f.state + // The grants part's list is not handed to the vault: the handing is of what the controller's list loaded (clause 9). + w.Handed = prev.Handed f.mod, f.state = file.module, w f.written = err == nil f.following = delivery.FollowGrants @@ -429,6 +515,38 @@ func (f *Follower) settleGrants(ctx context.Context, again bool) { } } +// Hold is c as the apply may write it (C3 of the switch's review): while the grants part's list is followed, the bus +// module's composition from the controller carries the controller's list, so its user-list resource holds the list on the +// machine instead (made 0600), or is left out when none can be read. Any other composition, or the controller's list +// followed, is c unchanged. +func (f *Follower) Hold(c delivery.Composed) delivery.Composed { + if f.Following() != delivery.FollowGrants { + return c + } + out := c + out.Resources = nil + for _, raw := range c.Resources { + if r, _, _, ok := userList([]json.RawMessage{raw}); ok { + var held json.RawMessage + if body, err := f.env.OnDisk(r); err == nil && body != nil { + var m map[string]any + if json.Unmarshal(r, &m) == nil { + delete(m, "digest") + delete(m, "size") + m["content"], m["mode"] = string(body), "0600" + held, _ = json.Marshal(m) + } + } + if held != nil { + out.Resources = append(out.Resources, held) + } + continue + } + out.Resources = append(out.Resources, raw) + } + return out +} + // mode0600 is a user-list resource made mode 0600, whatever its composition said (ADR 0547 clause 7). func mode0600(raw json.RawMessage) (json.RawMessage, error) { if len(raw) == 0 { @@ -442,6 +560,42 @@ func mode0600(raw json.RawMessage) (json.RawMessage, error) { return json.Marshal(r) } +// hand is the handing of the list on the machine to the vault: the one before when it is of the same list and key, a +// new one sealed now otherwise, and none while the vault's key is not known or the list holds no hash. +func (f *Follower) hand(loaded []byte, before *delivery.AccountsHanded) *delivery.AccountsHanded { + if f.env.VaultKey == nil || loaded == nil { + return before + } + key := f.env.VaultKey() + if key == "" { + return before + } + from := Digest(loaded) + if before != nil && before.From == from && before.To == delivery.KeyFingerprint(key) { + return before + } + accounts := delivery.HandedAccountsOf(loaded) + if len(accounts) == 0 { + return nil + } + h, err := delivery.HandAccounts(accounts, key, from, f.env.Now()) + if err != nil { + f.sayOnce(fmt.Sprintf("the loaded list's hashes are not handed to the vault: %v", err)) + return before + } + f.env.Say(fmt.Sprintf("the hashes of the %d users of the bus's loaded list (%s) are handed to the vault, sealed to "+ + "its key %s, in the bus module's have", h.Users, short(from), short(h.To))) + return &h +} + +// handedKey is what a handing says that a report compares: its list and key. +func handedKey(h *delivery.AccountsHanded) string { + if h == nil { + return "" + } + return h.From + " " + h.To +} + // ControllerUser is the controller's own user in the list: a list without it would lock the controller out of the bus. const ControllerUser = "controller" diff --git a/internal/userlist/userlist_test.go b/internal/userlist/userlist_test.go index 9b7c505..fb25d99 100644 --- a/internal/userlist/userlist_test.go +++ b/internal/userlist/userlist_test.go @@ -30,29 +30,48 @@ var _, strangerKey = key() func fpOf(c string) string { return delivery.SourcePrefix2 + strings.Repeat(c, 64) } -// feed is a watch the test writes to: the keys there before the watch, Caught, then each put. +// feed is a watch the test writes to: the keys there before the watch, Caught, then each put. watching is closed once +// the watch began, so a put after it is never handed before the keys there before it. type feed struct { ch chan agent.Update rev uint64 first []agent.Update - // watched is closed once the watch caught up, so a later put is not read before the keys there before it. - watched chan struct{} - once sync.Once + // watching is closed once the watch caught up, so a later put is not read before the keys there before it. + watching chan struct{} + once sync.Once + // err refuses the watch, as the bus refuses a watch it grants nothing of. + err error } func (f *feed) Watch(context.Context, string) (<-chan agent.Update, error) { + if f.err != nil { + return nil, f.err + } for _, u := range f.first { f.ch <- u } f.ch <- agent.Update{Caught: true} f.once.Do(func() { - if f.watched != nil { - close(f.watched) + if f.watching != nil { + close(f.watching) } }) return f.ch, nil } +// hashed is a list whose users carry a hash of bcrypt's form, as the composer writes them. +func hashed(users ...string) string { + var b strings.Builder + b.WriteString("authorization {\n users = [\n") + for _, u := range users { + b.WriteString(` { user: "` + u + `", password: "$2a$11$` + strings.Repeat("h", 53) + `", permissions: {` + "\n") + } + b.WriteString(" ]\n}\n") + return b.String() +} + +var vault, _ = seal.GenerateKeyPair() + func list(users ...string) string { var b strings.Builder b.WriteString("authorization {\n users = [\n") @@ -111,6 +130,8 @@ func comp(t *testing.T, module, fp, content string, signer ed25519.PrivateKey, p type gput struct { follow, content string otherKey, badDigest bool + // after puts it once the grants part's list was written, as a hand-back comes after a take-over. + after bool } type put struct { @@ -135,8 +156,17 @@ func TestTheBusNodeWritesTheNewestSignedUserListAlone(t *testing.T) { wantErr string wantRefused bool grants []gput + // grantsRefused refuses the watch of the grants part's key; kept is the key last heard, kept on the machine; + // wantUnwatched says the have then says why it is not watched. + grantsRefused bool + kept *gput + wantUnwatched bool // wantSource and wantFollows are what the last have says it took the list from and follows. wantSource, wantFollows string + // vault gives the follower the vault's key; wantHanded is how many users it then hands from the list on the + // machine (ADR 0547 clause 9). + vault bool + wantHanded int }{ {name: "the list there is written once", before: []put{{module: "nats", fp: fpOf("a"), content: two}}, wantWrites: []string{two}, wantUsers: 3}, @@ -170,36 +200,62 @@ func TestTheBusNodeWritesTheNewestSignedUserListAlone(t *testing.T) { onDisk: five, wantUsers: 2, wantErr: "drops more than half", wantRefused: true}, {name: "a list that drops half is written", before: []put{{module: "nats", fp: fpOf("a"), content: two}}, onDisk: list("controller", "a", "b", "c", "d", "e"), wantWrites: []string{two}, wantUsers: 3}, + {name: "the loaded list's hashes are handed sealed to the vault's key", + before: []put{{module: "nats", fp: fpOf("a"), content: hashed("controller", "node.ace")}}, vault: true, + wantWrites: []string{hashed("controller", "node.ace")}, wantUsers: 2, wantHanded: 2}, + {name: "a refused list hands the list on the machine", + before: []put{{module: "nats", fp: fpOf("a"), content: list("a", "b")}}, vault: true, + onDisk: hashed("controller", "a", "b"), wantUsers: 2, wantErr: "holds no user", wantRefused: true, wantHanded: 3}, {name: "a failed write is said and kept", before: []put{{module: "nats", fp: fpOf("a"), content: two}}, failWrite: true, wantWrites: []string{two}, wantUsers: 3, wantErr: "disk full"}, // ADR 0547 clauses 7, 8: never both lists; the grants part's from take-over, the controller's otherwise. {name: "from take-over the grants part's list is written into the same file", before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grants: []gput{{follow: delivery.FollowGrants, content: three}}, - wantWrites: []string{two, three}, wantUsers: 4, wantSource: delivery.GrantsUserListSource, wantFollows: delivery.FollowGrants}, + wantWrites: []string{three}, wantUsers: 4, wantSource: delivery.GrantsUserListSource, wantFollows: delivery.FollowGrants}, + {name: "a hand-back writes the newest controller composition once", + before: []put{{module: "nats", fp: fpOf("a"), content: two}}, + grants: []gput{{follow: delivery.FollowGrants, content: three}, {follow: delivery.FollowController, content: three, after: true}}, + wantWrites: []string{three, two}, wantUsers: 3, wantSource: "novox.nats." + fpOf("a")[len(delivery.SourcePrefix2):], wantFollows: delivery.FollowController}, + {name: "a refused watch with no key kept follows the controller's list and says it is not watched", + before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grantsRefused: true, wantUnwatched: true, + wantWrites: []string{two}, wantUsers: 3, wantSource: "novox.nats." + fpOf("a")[len(delivery.SourcePrefix2):], wantFollows: delivery.FollowController}, + {name: "a refused watch keeps the follow last heard, never the controller's list", + before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grantsRefused: true, wantUnwatched: true, + kept: &gput{follow: delivery.FollowGrants, content: three}, + wantWrites: []string{three}, wantUsers: 4, wantSource: delivery.GrantsUserListSource, wantFollows: delivery.FollowGrants}, {name: "in shadow the grants part's list is not taken", before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grants: []gput{{follow: delivery.FollowController, content: three}}, wantWrites: []string{two}, wantUsers: 3, wantSource: "novox.nats." + fpOf("a")[len(delivery.SourcePrefix2):], wantFollows: delivery.FollowController}, {name: "a grants list that does not open to its digest is refused and the file kept", before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grants: []gput{{follow: delivery.FollowGrants, content: three, badDigest: true}}, - onDisk: two, wantWrites: []string{two}, wantErr: "could not be had", wantRefused: true, wantSource: delivery.GrantsUserListSource}, + onDisk: two, wantErr: "could not be had", wantRefused: true, wantSource: delivery.GrantsUserListSource}, {name: "a grants list sealed to another key is refused and the file kept", before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grants: []gput{{follow: delivery.FollowGrants, content: three, otherKey: true}}, - onDisk: two, wantWrites: []string{two}, wantErr: "sealed to another key", wantRefused: true, wantSource: delivery.GrantsUserListSource}, + onDisk: two, wantErr: "sealed to another key", wantRefused: true, wantSource: delivery.GrantsUserListSource}, } { t.Run(c.name, func(t *testing.T) { - f := &feed{ch: make(chan agent.Update, 64), watched: make(chan struct{})} + f := &feed{ch: make(chan agent.Update, 64), watching: make(chan struct{})} objects := map[string]string{} var mu sync.Mutex var writes []string var reports []Written reported := make(chan struct{}, 64) gf := &feed{ch: make(chan agent.Update, 64)} + if c.grantsRefused { + gf.err = errors.New("Permissions Violation for Publish to $JS.API.STREAM.INFO.KV_mesh-grants_user-list") + } + var grantsAfter []agent.Update + var keptRaw []byte sealed := map[string][]byte{} nodeKey, err := seal.GenerateKeyPair() if err != nil { t.Fatal(err) } - for i, g := range c.grants { + gs := c.grants + if c.kept != nil { + gs = append(append([]gput(nil), gs...), *c.kept) + } + for i, g := range gs { to := nodeKey.PublicKey if g.otherKey { other, _ := seal.GenerateKeyPair() @@ -219,9 +275,20 @@ func TestTheBusNodeWritesTheNewestSignedUserListAlone(t *testing.T) { if err != nil { t.Fatal(err) } - gf.first = append(gf.first, agent.Update{Key: delivery.GrantsUserListKey, Value: raw, Revision: uint64(i + 1)}) + u := agent.Update{Key: delivery.GrantsUserListKey, Value: raw, Revision: uint64(i + 1)} + switch { + case c.kept != nil && i == len(gs)-1: + keptRaw = raw + case g.after: + grantsAfter = append(grantsAfter, u) + default: + gf.first = append(gf.first, u) + } } fl := New(Env{Node: node, Composed: f, Grants: gf, + Kept: func() ([]byte, uint64) { return keptRaw, 9 }, + Keep: func([]byte, uint64) {}, + Backoff: func(int) time.Duration { return time.Hour }, Fetch: func(_ context.Context, name string) ([]byte, error) { if o, ok := sealed[name]; ok { return o, nil @@ -268,6 +335,12 @@ func TestTheBusNodeWritesTheNewestSignedUserListAlone(t *testing.T) { reported <- struct{}{} }, Now: func() time.Time { return time.Date(2026, 10, 25, 21, 50, 0, 0, time.UTC) }, + VaultKey: func() string { + if c.vault { + return vault.PublicKey + } + return "" + }, }) update := func(p put) agent.Update { f.rev++ @@ -291,10 +364,31 @@ func TestTheBusNodeWritesTheNewestSignedUserListAlone(t *testing.T) { defer cancel() done := make(chan error, 1) go func() { done <- fl.Run(ctx) }() - <-f.watched + <-f.watching + // A later put after the first list is written: the first write waits for the grants part's key to catch up. + if len(c.after) > 0 { + for until := time.Now().Add(5 * time.Second); time.Now().Before(until); time.Sleep(5 * time.Millisecond) { + mu.Lock() + n := len(writes) + mu.Unlock() + if n > 0 { + break + } + } + } for _, p := range c.after { f.ch <- update(p) } + if len(grantsAfter) > 0 { + for until := time.Now().Add(5 * time.Second); ; time.Sleep(5 * time.Millisecond) { + if _, st := fl.State(); st.Source == delivery.GrantsUserListSource || time.Now().After(until) { + break + } + } + for _, u := range grantsAfter { + gf.ch <- u + } + } if c.again { // After the first write is reported, so the look again is not folded into it. select { @@ -341,10 +435,41 @@ func TestTheBusNodeWritesTheNewestSignedUserListAlone(t *testing.T) { t.Fatalf("written %q, want %q", writes, c.wantWrites) } module, st := fl.State() + if c.wantUnwatched != (st.Unwatched != "") { + t.Fatalf("the have says unwatched %q", st.Unwatched) + } + // While the grants part's list is followed, an apply of the controller's composition writes not its list. + if _, raw := comp(t, "nats", fpOf("a"), two, controllerKey, put{}); c.wantFollows != "" { + cmp, err := delivery.DecodeComposed(raw) + if err != nil { + t.Fatal(err) + } + held := fl.Hold(cmp) + _, _, _, hasList := userList(held.Resources) + if want := c.wantFollows == delivery.FollowController; hasList != want || len(held.Resources) < 1 { + t.Fatalf("an apply while following %s holds the controller's list: %v", c.wantFollows, hasList) + } + } if c.wantSource != "" && (st.Source != c.wantSource || st.Key != nodeKey.PublicKey || (c.wantFollows != "" && st.Follows != c.wantFollows)) { t.Fatalf("the have says source %q follows %q key %t; want %q %q", st.Source, st.Follows, st.Key == nodeKey.PublicKey, c.wantSource, c.wantFollows) } + switch { + case c.wantHanded == 0 && st.Handed != nil: + t.Fatalf("handed with no vault key or no hash: %+v", st.Handed) + case c.wantHanded > 0: + loaded := c.onDisk + if !c.wantRefused { + loaded = c.wantWrites[len(c.wantWrites)-1] + } + if st.Handed == nil || st.Handed.From != Digest([]byte(loaded)) { + t.Fatalf("handed %+v, want from the list on the machine", st.Handed) + } + opened, err := delivery.OpenHanded(*st.Handed, vault.PublicKey, vault.PrivateKey) + if err != nil || len(opened) != c.wantHanded || opened[0].User != "controller" { + t.Fatalf("opened %+v, %v", opened, err) + } + } if c.wantRefused { if !strings.Contains(st.Error, c.wantErr) || st.Digest != Digest([]byte(c.onDisk)) || st.Users != c.wantUsers { t.Fatalf("refused list's state %+v", st)