From fd381567f4a2553a63762895ae55a978363dac4b Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 11:32:36 +0100 Subject: [PATCH 1/6] The bus node hands the loaded list's hashes to the vault, sealed to its key, in the bus module's have (ADR 0547 clause 9, issue 1945) The vault holds no value for the controller's own account, the node accounts and the enrolled devices, so it cannot hash them, and take-over waits on their hashes. The bus node's engine is the one part that has them, in the list the bus loaded. Its follower of the user list now puts every loaded user's bcrypt hash in the bus module's have, sealed to the vault's own key (mesh-sdk delivery.AccountsHanded), so of every reader of the have bucket the vault alone opens it; the clear part says only the list's digest, the key's fingerprint and how many users. The vault's key is heard from its answer to node-key, which the engine already asks at start; a first or changed key looks at the list again. It is sealed again only when the list on the machine or the vault's key changes; a refused list hands the list kept on the machine. The vault takes from it only the users it holds no value for. Also: the user list follower's test sent its later puts before the watch had handed the keys there before it, so two of its cases failed on main whenever the goroutine started late; the feed now says when the watch began. --- cmd/mesh-host/main.go | 2 +- cmd/mesh-host/nodeagent.go | 2 + cmd/mesh-host/nodekey.go | 61 ++++++++++++++++++++++++++++-- cmd/mesh-host/nodekey_test.go | 30 ++++++++++++++- cmd/mesh-host/userlist.go | 5 ++- go.mod | 2 +- go.sum | 2 + internal/userlist/userlist.go | 60 +++++++++++++++++++++++++++-- internal/userlist/userlist_test.go | 60 ++++++++++++++++++++++++++--- 9 files changed, 208 insertions(+), 16 deletions(-) diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index 06ebfd9..8515ea8 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -1153,7 +1153,7 @@ func runLink(ctx context.Context, opts options) error { return identity.LoadSealingKey(identity.SealingKeyPath(opts.state)) }, func(verb string, args any) (json.RawMessage, error) { return queue.AskVault(ctx, verb, args) - }, say, nodeKeyRetryFrom, nodeKeyRetryUpTo) + }, say, nodeKeyRetryFrom, nodeKeyRetryUpTo, vaultKey.heard) // And whether what it runs stays up, looked at on its own clock and said when it changes (novox/hq // ADR 0240) — between applies, which is when a container crash-loops. if judging != nil { diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index 8c47ac4..224dfbc 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -412,6 +412,8 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess } }) }) + // A vault key heard (or heard again changed) looks at the list again, so its hashes are handed (ADR 0547 §9). + vaultKey.following(users.Again) go func() { if err := users.Run(linked); err != nil && linked.Err() == nil { say("the bus's user list is not followed on this link: " + err.Error()) diff --git a/cmd/mesh-host/nodekey.go b/cmd/mesh-host/nodekey.go index f00c2f8..18fe355 100644 --- a/cmd/mesh-host/nodekey.go +++ b/cmd/mesh-host/nodekey.go @@ -2,7 +2,10 @@ package main import ( "context" + "encoding/json" "fmt" + "strings" + "sync" "time" "github.com/novox/mesh-host/internal/identity" @@ -22,11 +25,60 @@ const ( nodeKeyLook = 30 * time.Second ) +// heardKey is the vault's own public key as its answer to node-key gave it (ADR 0547 clause 9, issue 1945): the key the +// bus node hands the loaded list's hashes sealed to. One for the process, as the key stated is; kick is the follower of +// the user list on the link open now, told when the key is first heard or changes. +type heardKey struct { + mu sync.Mutex + pem string + kick func() +} + +// vaultKey is the vault's key this engine heard. +var vaultKey = &heardKey{} + +func (h *heardKey) get() string { + h.mu.Lock() + defer h.mu.Unlock() + return h.pem +} + +// heard keeps the key a node-key answer gave, and tells the follower when it is a first or changed one. An answer +// with no key (a vault before this one) changes nothing. +func (h *heardKey) heard(answer json.RawMessage) { + var a struct { + VaultKey string `json:"vault_key"` + } + if json.Unmarshal(answer, &a) != nil || !strings.Contains(a.VaultKey, "PUBLIC KEY") { + return + } + h.mu.Lock() + changed := a.VaultKey != h.pem + h.pem = a.VaultKey + kick := h.kick + h.mu.Unlock() + if changed && kick != nil { + kick() + } +} + +// following names the follower told when the key changes, and tells it at once when a key is known. +func (h *heardKey) following(kick func()) { + h.mu.Lock() + h.kick = kick + known := h.pem != "" + h.mu.Unlock() + if known && kick != nil { + kick() + } +} + // keepNodeKeyStated states the sealing key load answers to the vault at start, retrying with a growing wait from // retryFrom to retryUpTo while it does not answer, and states it again whenever the key changes. It says one line -// when a key is stated and one when a try fails for another reason than the last, and returns when ctx ends. +// when a key is stated and one when a try fails for another reason than the last, and returns when ctx ends. Each +// answer goes to heard, which takes the vault's own key from it. func keepNodeKeyStated(ctx context.Context, load func() (identity.SealingKey, error), ask vaultAsk, say func(string), - retryFrom, retryUpTo time.Duration) { + retryFrom, retryUpTo time.Duration, heard func(json.RawMessage)) { stated, lastFailure := "", "" wait := retryFrom fail := func(why string) { @@ -45,10 +97,13 @@ func keepNodeKeyStated(ctx context.Context, load func() (identity.SealingKey, er fail(err.Error()) delay = wait case key.Public != stated: - if _, err := ask("node-key", map[string]string{"key": key.Public}); err != nil { + if answer, err := ask("node-key", map[string]string{"key": key.Public}); err != nil { fail(err.Error()) delay = wait } else { + if heard != nil { + heard(answer) + } stated, lastFailure, wait = key.Public, "", retryFrom say(fmt.Sprintf("mesh-host: this machine's sealing key %s is stated to the vault", key.Public)) } diff --git a/cmd/mesh-host/nodekey_test.go b/cmd/mesh-host/nodekey_test.go index 80f85ce..284a326 100644 --- a/cmd/mesh-host/nodekey_test.go +++ b/cmd/mesh-host/nodekey_test.go @@ -71,9 +71,14 @@ func TestTheEngineStatesItsSealingKeyToTheVaultAtStartRetriesAndRestatesAChange( say := func(l string) { lmu.Lock(); lines = append(lines, l); lmu.Unlock() } ctx, cancel := context.WithCancel(context.Background()) defer cancel() - go keepNodeKeyStated(ctx, load, vault.ask, say, time.Millisecond, 5*time.Millisecond) + var hmu sync.Mutex + var answers []string + heard := func(a json.RawMessage) { hmu.Lock(); answers = append(answers, string(a)); hmu.Unlock() } + go keepNodeKeyStated(ctx, load, vault.ask, say, time.Millisecond, 5*time.Millisecond, heard) waitFor(t, "the first key stated after two refusals", func() bool { return len(vault.said()) == 1 }) + // Its answer is handed on, so the vault's own key in it is heard (ADR 0547 clause 9); refusals hand nothing. + waitFor(t, "the answer heard", func() bool { hmu.Lock(); defer hmu.Unlock(); return len(answers) == 1 }) if vault.said()[0] != first.Public { t.Errorf("stated %q, want the engine's own key %q", vault.said()[0], first.Public) } @@ -104,9 +109,30 @@ func TestTheEngineStatesItsSealingKeyToTheVaultAtStartRetriesAndRestatesAChange( } } -// The statement a host makes names this contract (issue 786), one above the vault's box. +// The statement a host makes names this contract (issue 786), one above the vault's box. And the vault's key its +// answer gives is kept, told to the follower once when first heard and again only when changed; an answer without one +// (an older vault) or with no public key changes nothing (ADR 0547 clause 9). func TestTheNodeKeyContractIsThirteen(t *testing.T) { if link.NodeKeyContract != 13 { t.Errorf("NodeKeyContract %d", link.NodeKeyContract) } + h := &heardKey{} + kicks := 0 + h.following(func() { kicks++ }) + h.heard(json.RawMessage(`{"recorded":true}`)) + h.heard(json.RawMessage(`{"recorded":true,"vault_key":"not a key"}`)) + if h.get() != "" || kicks != 0 { + t.Fatalf("an answer with no key was kept: %q, %d kicks", h.get(), kicks) + } + pem := "-----BEGIN PUBLIC KEY-----\nAAAA\n-----END PUBLIC KEY-----\n" + answer, _ := json.Marshal(map[string]any{"recorded": true, "vault_key": pem}) + h.heard(answer) + h.heard(answer) + if h.get() != pem || kicks != 1 { + t.Fatalf("key %q, %d kicks, want the key and one", h.get(), kicks) + } + h.following(func() { kicks += 10 }) + if kicks != 11 { + t.Fatalf("a follower named after the key was heard was not told: %d", kicks) + } } diff --git a/cmd/mesh-host/userlist.go b/cmd/mesh-host/userlist.go index ff5527a..642074e 100644 --- a/cmd/mesh-host/userlist.go +++ b/cmd/mesh-host/userlist.go @@ -42,7 +42,10 @@ func userListFollower(opts options, session *link.Session, composedBucket string Report: func(ctx context.Context, module string, w userlist.Written) { session.NoteUserList(ctx, module, w) }, - Say: announceOr(say), + // The vault's key from its answer to node-key: the loaded list's hashes are handed sealed to it (ADR 0547 + // clause 9). + VaultKey: vaultKey.get, + Say: announceOr(say), }) } diff --git a/go.mod b/go.mod index c100297..f096840 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/novox/mesh-host go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025204730-0be0ffd6159e + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026102838-174789941936 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 golang.org/x/crypto v0.57.0 diff --git a/go.sum b/go.sum index f843500..bb8a904 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025204213-f63fa86c7456 h1:He2o9vSK git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025204213-f63fa86c7456/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025204730-0be0ffd6159e h1:k/1NjjW5HuuuHW2ldyWTvjp+1u3fN0RO18Q57Iup6Dk= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025204730-0be0ffd6159e/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026102838-174789941936 h1:HGBoWimZXYZp+Qn9VCstYDitOP84t9EUQYd1yM9j2eg= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026102838-174789941936/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/internal/userlist/userlist.go b/internal/userlist/userlist.go index 7992a6e..1873ed7 100644 --- a/internal/userlist/userlist.go +++ b/internal/userlist/userlist.go @@ -15,6 +15,11 @@ // older list, and when asked (Again). // // It reads no want and writes none, moves no fingerprint, and touches nothing of the bus module but its user list. +// +// **It hands the loaded list's hashes to the vault** (ADR 0547 clause 9, issue 1945): once the vault's key is known +// (VaultKey), the have of the bus module carries the bcrypt hash of every user of the list on the machine, sealed to +// that key (delivery.AccountsHanded), so the vault adopts the accounts it holds no value for. It is sealed again only +// when the list on the machine or the vault's key changes. package userlist import ( @@ -59,8 +64,11 @@ type Env struct { Write func(ctx context.Context, module string, resource json.RawMessage, body []byte) (digest string, changed bool, err error) // Report says what was written, in the module's have. Report func(ctx context.Context, module string, w Written) - Now func() time.Time - Say func(string) + // VaultKey is the vault's own public key (SPKI PEM) as its answer to node-key gave it, "" while none is known: the + // key the loaded list's hashes are handed sealed to. Nil hands nothing. + VaultKey func() string + Now func() time.Time + Say func(string) } type source struct { @@ -220,14 +228,24 @@ func (f *Follower) settle(ctx context.Context, again bool) { } } } + var loaded []byte if err == nil { w.Digest, changed, err = f.env.Write(ctx, newest.module, newest.resource, body) + if err == nil { + loaded = body + } + } else if body != nil && f.env.OnDisk != nil { + // Refused: the list on the machine stays, and it is the one the bus has loaded. + loaded, _ = f.env.OnDisk(newest.resource) } if err != nil { w.Error = err.Error() } f.mu.Lock() prev := f.state + f.mu.Unlock() + w.Handed = f.hand(loaded, prev.Handed) + f.mu.Lock() f.mod, f.state = newest.module, w f.written = err == nil f.mu.Unlock() @@ -239,11 +257,47 @@ func (f *Follower) settle(ctx context.Context, again bool) { } // Reported when what it says changed: a look again that found the file as it was says nothing new. if f.env.Report != nil && (changed || prev.Source != w.Source || prev.Revision != w.Revision || - prev.Digest != w.Digest || prev.Error != w.Error) { + prev.Digest != w.Digest || prev.Error != w.Error || handedKey(prev.Handed) != handedKey(w.Handed)) { f.env.Report(ctx, newest.module, w) } } +// hand is the handing of the list on the machine to the vault: the one before when it is of the same list and key, a +// new one sealed now otherwise, and none while the vault's key is not known or the list holds no hash. +func (f *Follower) hand(loaded []byte, before *delivery.AccountsHanded) *delivery.AccountsHanded { + if f.env.VaultKey == nil || loaded == nil { + return before + } + key := f.env.VaultKey() + if key == "" { + return before + } + from := Digest(loaded) + if before != nil && before.From == from && before.To == delivery.KeyFingerprint(key) { + return before + } + accounts := delivery.HandedAccountsOf(loaded) + if len(accounts) == 0 { + return nil + } + h, err := delivery.HandAccounts(accounts, key, from, f.env.Now()) + if err != nil { + f.sayOnce(fmt.Sprintf("the loaded list's hashes are not handed to the vault: %v", err)) + return before + } + f.env.Say(fmt.Sprintf("the hashes of the %d users of the bus's loaded list (%s) are handed to the vault, sealed to "+ + "its key %s, in the bus module's have", h.Users, short(from), short(h.To))) + return &h +} + +// handedKey is what a handing says that a report compares: its list and key. +func handedKey(h *delivery.AccountsHanded) string { + if h == nil { + return "" + } + return h.From + " " + h.To +} + // ControllerUser is the controller's own user in the list: a list without it would lock the controller out of the bus. const ControllerUser = "controller" diff --git a/internal/userlist/userlist_test.go b/internal/userlist/userlist_test.go index d4d52c1..55f8080 100644 --- a/internal/userlist/userlist_test.go +++ b/internal/userlist/userlist_test.go @@ -11,6 +11,7 @@ import ( "time" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/seal" "github.com/novox/mesh-host/internal/agent" ) @@ -29,11 +30,13 @@ var _, strangerKey = key() func fpOf(c string) string { return delivery.SourcePrefix2 + strings.Repeat(c, 64) } -// feed is a watch the test writes to: the keys there before the watch, Caught, then each put. +// feed is a watch the test writes to: the keys there before the watch, Caught, then each put. watching is closed once +// the watch began, so a put after it is never handed before the keys there before it. type feed struct { - ch chan agent.Update - rev uint64 - first []agent.Update + ch chan agent.Update + rev uint64 + first []agent.Update + watching chan struct{} } func (f *feed) Watch(context.Context, string) (<-chan agent.Update, error) { @@ -41,9 +44,23 @@ func (f *feed) Watch(context.Context, string) (<-chan agent.Update, error) { f.ch <- u } f.ch <- agent.Update{Caught: true} + close(f.watching) return f.ch, nil } +// hashed is a list whose users carry a hash of bcrypt's form, as the composer writes them. +func hashed(users ...string) string { + var b strings.Builder + b.WriteString("authorization {\n users = [\n") + for _, u := range users { + b.WriteString(` { user: "` + u + `", password: "$2a$11$` + strings.Repeat("h", 53) + `", permissions: {` + "\n") + } + b.WriteString(" ]\n}\n") + return b.String() +} + +var vault, _ = seal.GenerateKeyPair() + func list(users ...string) string { var b strings.Builder b.WriteString("authorization {\n users = [\n") @@ -118,6 +135,10 @@ func TestTheBusNodeWritesTheNewestSignedUserListAlone(t *testing.T) { wantUsers int wantErr string wantRefused bool + // vault gives the follower the vault's key; wantHanded is how many users it then hands from the list on the + // machine (ADR 0547 clause 9). + vault bool + wantHanded int }{ {name: "the list there is written once", before: []put{{module: "nats", fp: fpOf("a"), content: two}}, wantWrites: []string{two}, wantUsers: 3}, @@ -151,11 +172,17 @@ func TestTheBusNodeWritesTheNewestSignedUserListAlone(t *testing.T) { onDisk: five, wantUsers: 2, wantErr: "drops more than half", wantRefused: true}, {name: "a list that drops half is written", before: []put{{module: "nats", fp: fpOf("a"), content: two}}, onDisk: list("controller", "a", "b", "c", "d", "e"), wantWrites: []string{two}, wantUsers: 3}, + {name: "the loaded list's hashes are handed sealed to the vault's key", + before: []put{{module: "nats", fp: fpOf("a"), content: hashed("controller", "node.ace")}}, vault: true, + wantWrites: []string{hashed("controller", "node.ace")}, wantUsers: 2, wantHanded: 2}, + {name: "a refused list hands the list on the machine", + before: []put{{module: "nats", fp: fpOf("a"), content: list("a", "b")}}, vault: true, + onDisk: hashed("controller", "a", "b"), wantUsers: 2, wantErr: "holds no user", wantRefused: true, wantHanded: 3}, {name: "a failed write is said and kept", before: []put{{module: "nats", fp: fpOf("a"), content: two}}, failWrite: true, wantWrites: []string{two}, wantUsers: 3, wantErr: "disk full"}, } { t.Run(c.name, func(t *testing.T) { - f := &feed{ch: make(chan agent.Update, 64)} + f := &feed{ch: make(chan agent.Update, 64), watching: make(chan struct{})} objects := map[string]string{} var mu sync.Mutex var writes []string @@ -197,6 +224,12 @@ func TestTheBusNodeWritesTheNewestSignedUserListAlone(t *testing.T) { reported <- struct{}{} }, Now: func() time.Time { return time.Date(2026, 10, 25, 21, 50, 0, 0, time.UTC) }, + VaultKey: func() string { + if c.vault { + return vault.PublicKey + } + return "" + }, }) update := func(p put) agent.Update { f.rev++ @@ -220,6 +253,7 @@ func TestTheBusNodeWritesTheNewestSignedUserListAlone(t *testing.T) { defer cancel() done := make(chan error, 1) go func() { done <- fl.Run(ctx) }() + <-f.watching for _, p := range c.after { f.ch <- update(p) } @@ -261,6 +295,22 @@ func TestTheBusNodeWritesTheNewestSignedUserListAlone(t *testing.T) { t.Fatalf("written %q, want %q", writes, c.wantWrites) } module, st := fl.State() + switch { + case c.wantHanded == 0 && st.Handed != nil: + t.Fatalf("handed with no vault key or no hash: %+v", st.Handed) + case c.wantHanded > 0: + loaded := c.onDisk + if !c.wantRefused { + loaded = c.wantWrites[len(c.wantWrites)-1] + } + if st.Handed == nil || st.Handed.From != Digest([]byte(loaded)) { + t.Fatalf("handed %+v, want from the list on the machine", st.Handed) + } + opened, err := delivery.OpenHanded(*st.Handed, vault.PublicKey, vault.PrivateKey) + if err != nil || len(opened) != c.wantHanded || opened[0].User != "controller" { + t.Fatalf("opened %+v, %v", opened, err) + } + } if c.wantRefused { if !strings.Contains(st.Error, c.wantErr) || st.Digest != Digest([]byte(c.onDisk)) || st.Users != c.wantUsers { t.Fatalf("refused list's state %+v", st) -- 2.54.0 From 1a190090ab8a09ff11f9288b71e49880220cc1cd Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 12:03:12 +0100 Subject: [PATCH 2/6] The engine states its key again until it hears the vault's key, and at a slow interval after, so a vault delivered after it or a changed vault key is heard (review of the adoption handing, ADR 0547 clause 9) --- cmd/mesh-host/main.go | 3 +- cmd/mesh-host/nodekey.go | 39 +++++++++++++++++++----- cmd/mesh-host/nodekey_test.go | 56 +++++++++++++++++++++++++++++++++-- 3 files changed, 88 insertions(+), 10 deletions(-) diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index 8515ea8..ade268e 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -1153,7 +1153,8 @@ func runLink(ctx context.Context, opts options) error { return identity.LoadSealingKey(identity.SealingKeyPath(opts.state)) }, func(verb string, args any) (json.RawMessage, error) { return queue.AskVault(ctx, verb, args) - }, say, nodeKeyRetryFrom, nodeKeyRetryUpTo, vaultKey.heard) + }, say, nodeKeyRetryFrom, nodeKeyRetryUpTo, &keyHearing{heard: vaultKey.heard, + held: func() bool { return vaultKey.get() != "" }, look: nodeKeyLook, every: nodeKeyReask}) // And whether what it runs stays up, looked at on its own clock and said when it changes (novox/hq // ADR 0240) — between applies, which is when a container crash-loops. if judging != nil { diff --git a/cmd/mesh-host/nodekey.go b/cmd/mesh-host/nodekey.go index 18fe355..5881baf 100644 --- a/cmd/mesh-host/nodekey.go +++ b/cmd/mesh-host/nodekey.go @@ -73,13 +73,29 @@ func (h *heardKey) following(kick func()) { } } +// nodeKeyReask is how often the key is stated again though it did not change, so a changed vault key is heard: the +// vault takes the same key again and changes nothing (ADR 0547 clause 9). +const nodeKeyReask = 10 * time.Minute + +// keyHearing is what the keeper does with the vault's own key in node-key's answers (ADR 0547 clause 9): heard takes +// it from an answer; held says whether one is held, and while none is the key is stated again at every look, so a vault +// delivered after this engine is heard within a look; every is how often it is stated again all the same, so a changed +// vault key is heard. +type keyHearing struct { + heard func(json.RawMessage) + held func() bool + // look is how often it is stated again while no vault key is held; every how often once one is. + look, every time.Duration +} + // keepNodeKeyStated states the sealing key load answers to the vault at start, retrying with a growing wait from // retryFrom to retryUpTo while it does not answer, and states it again whenever the key changes. It says one line -// when a key is stated and one when a try fails for another reason than the last, and returns when ctx ends. Each -// answer goes to heard, which takes the vault's own key from it. +// when a key is stated and one when a try fails for another reason than the last, and returns when ctx ends. With +// hearing, each answer goes to it, and the key is stated again while no vault key is held and at its interval. func keepNodeKeyStated(ctx context.Context, load func() (identity.SealingKey, error), ask vaultAsk, say func(string), - retryFrom, retryUpTo time.Duration, heard func(json.RawMessage)) { + retryFrom, retryUpTo time.Duration, hearing *keyHearing) { stated, lastFailure := "", "" + var askedAt time.Time wait := retryFrom fail := func(why string) { if why != lastFailure { @@ -92,20 +108,29 @@ func keepNodeKeyStated(ctx context.Context, load func() (identity.SealingKey, er for { delay := look key, err := load() + again := false + if hearing != nil && stated != "" { + since := time.Since(askedAt) + held := hearing.held == nil || hearing.held() + again = (!held && since >= hearing.look) || (held && hearing.every > 0 && since >= hearing.every) + } switch { case err != nil: fail(err.Error()) delay = wait - case key.Public != stated: + case key.Public != stated || again: if answer, err := ask("node-key", map[string]string{"key": key.Public}); err != nil { fail(err.Error()) delay = wait } else { - if heard != nil { - heard(answer) + askedAt = time.Now() + if hearing != nil && hearing.heard != nil { + hearing.heard(answer) + } + if key.Public != stated { + say(fmt.Sprintf("mesh-host: this machine's sealing key %s is stated to the vault", key.Public)) } stated, lastFailure, wait = key.Public, "", retryFrom - say(fmt.Sprintf("mesh-host: this machine's sealing key %s is stated to the vault", key.Public)) } } select { diff --git a/cmd/mesh-host/nodekey_test.go b/cmd/mesh-host/nodekey_test.go index 284a326..0bae9f5 100644 --- a/cmd/mesh-host/nodekey_test.go +++ b/cmd/mesh-host/nodekey_test.go @@ -19,6 +19,10 @@ type keyVault struct { stated []string fail int verbs []string + // keyless is how many answers carry no vault key (a vault before ADR 0547 clause 9's, or not yet started); key the + // vault's key the others carry. + keyless int + key string } func (v *keyVault) ask(verb string, args any) (json.RawMessage, error) { @@ -35,7 +39,12 @@ func (v *keyVault) ask(verb string, args any) (json.RawMessage, error) { } _ = json.Unmarshal(raw, &a) v.stated = append(v.stated, a.Key) - return json.RawMessage(`{"recorded":true}`), nil + if v.key == "" || v.keyless > 0 { + v.keyless-- + return json.RawMessage(`{"recorded":true}`), nil + } + answer, _ := json.Marshal(map[string]any{"recorded": true, "vault_key": v.key}) + return answer, nil } func (v *keyVault) said() []string { @@ -74,7 +83,7 @@ func TestTheEngineStatesItsSealingKeyToTheVaultAtStartRetriesAndRestatesAChange( var hmu sync.Mutex var answers []string heard := func(a json.RawMessage) { hmu.Lock(); answers = append(answers, string(a)); hmu.Unlock() } - go keepNodeKeyStated(ctx, load, vault.ask, say, time.Millisecond, 5*time.Millisecond, heard) + go keepNodeKeyStated(ctx, load, vault.ask, say, time.Millisecond, 5*time.Millisecond, &keyHearing{heard: heard}) waitFor(t, "the first key stated after two refusals", func() bool { return len(vault.said()) == 1 }) // Its answer is handed on, so the vault's own key in it is heard (ADR 0547 clause 9); refusals hand nothing. @@ -107,6 +116,49 @@ func TestTheEngineStatesItsSealingKeyToTheVaultAtStartRetriesAndRestatesAChange( if len(lines) == 0 { t.Error("the engine said nothing") } + asksAgainUntilItHearsTheVaultsKey(t) +} + +// ADR 0547 clause 9: the vault's key is heard though the engine's own key never changes. While none is held the key +// is stated again at each look, so a vault delivered after the engine (or answering no key yet) is heard; once one is +// held, again only at the slow interval, so a changed vault key is heard; never more. +func asksAgainUntilItHearsTheVaultsKey(t *testing.T) { + sealing, _ := identity.GenerateSealingKey() + load := func() (identity.SealingKey, error) { return sealing, nil } + pem := func(c string) string { return "-----BEGIN PUBLIC KEY-----\n" + c + "\n-----END PUBLIC KEY-----\n" } + for _, c := range []struct { + name string + keyless int + every time.Duration + change bool + wantAtLeast int + wantAtMost int + }{ + {name: "a vault with no key yet is asked again until it answers one, then not", keyless: 3, wantAtLeast: 4, wantAtMost: 4}, + {name: "a held key is asked again at the slow interval, and a changed one heard", every: 20 * time.Millisecond, + change: true, wantAtLeast: 3, wantAtMost: 1 << 20}, + } { + t.Run(c.name, func(t *testing.T) { + vault := &keyVault{keyless: c.keyless, key: pem("AAAA")} + h := &heardKey{} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go keepNodeKeyStated(ctx, load, vault.ask, func(string) {}, time.Millisecond, 5*time.Millisecond, + &keyHearing{heard: h.heard, held: func() bool { return h.get() != "" }, look: time.Millisecond, every: c.every}) + waitFor(t, "the vault's key heard", func() bool { return h.get() == pem("AAAA") }) + if c.change { + vault.mu.Lock() + vault.key = pem("BBBB") + vault.mu.Unlock() + waitFor(t, "the changed vault key heard", func() bool { return h.get() == pem("BBBB") }) + } + time.Sleep(60 * time.Millisecond) + n := len(vault.said()) + if n < c.wantAtLeast || n > c.wantAtMost { + t.Errorf("node-key asked %d times, want %d to %d", n, c.wantAtLeast, c.wantAtMost) + } + }) + } } // The statement a host makes names this contract (issue 786), one above the vault's box. And the vault's key its -- 2.54.0 From 882e75c954ea1d5ece40e630dce03950fb7f56b7 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 12:06:17 +0100 Subject: [PATCH 3/6] The bus node follows the grants part's sealed user list from take-over, the controller's otherwise, never both (ADR 0547 clauses 7, 8; issue 1945) On the node whose compositions carry the bus's list, the engine also watches mesh-grants' key user-list.sealed. While it says follow grants, the object it names is read by name, opened with this machine's sealing key, checked against the key's digest and the guard, and written to the same file at 0600 only when its digest differs; the bus reloads it itself. The have says which list is followed and this machine's sealing key, which the grants part seals to. The follower's test waits for its watch to catch up before a later put, which raced under load. Pins mesh-sdk 175a5567. --- cmd/mesh-host/userlist.go | 30 +++++ go.mod | 2 +- go.sum | 2 + internal/link/files.go | 52 +++++++- internal/userlist/userlist.go | 208 ++++++++++++++++++++++++++++- internal/userlist/userlist_test.go | 85 +++++++++++- 6 files changed, 369 insertions(+), 10 deletions(-) diff --git a/cmd/mesh-host/userlist.go b/cmd/mesh-host/userlist.go index ff5527a..a0d58fc 100644 --- a/cmd/mesh-host/userlist.go +++ b/cmd/mesh-host/userlist.go @@ -9,7 +9,10 @@ import ( "os" "git.novox.be/novox/mesh-sdk/go/declaration" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/seal" "github.com/novox/mesh-host/internal/apply" + "github.com/novox/mesh-host/internal/identity" "github.com/novox/mesh-host/internal/link" "github.com/novox/mesh-host/internal/store" "github.com/novox/mesh-host/internal/system" @@ -42,10 +45,37 @@ func userListFollower(opts options, session *link.Session, composedBucket string Report: func(ctx context.Context, module string, w userlist.Written) { session.NoteUserList(ctx, module, w) }, + // From take-over, the grants part's list (ADR 0547 clauses 7, 8): its one key watched, its object read by name and + // opened with this machine's sealing key, the key's public half said in the have so the grants part seals to it. + Grants: bucketWatch{session, delivery.GrantsUserListBucket}, + Fetch: func(ctx context.Context, name string) ([]byte, error) { + return session.GrantsObject(ctx, name, grantsListLimit) + }, + Open: func(b seal.Box) ([]byte, error) { + key, err := identity.LoadSealingKey(identity.SealingKeyPath(opts.state)) + if err != nil { + return nil, err + } + return key.OpenVaultBox(b) + }, + Key: func() string { + key, err := identity.LoadSealingKey(identity.SealingKeyPath(opts.state)) + if err != nil { + return "" + } + pem, err := key.PublicPEM() + if err != nil { + return "" + } + return pem + }, Say: announceOr(say), }) } +// grantsListLimit is the largest sealed list the bus node reads: the live list is about a hundred kilobytes. +const grantsListLimit = 16 << 20 + // userListFile is a user-list resource as the file it declares. func userListFile(raw []byte) (*declaration.File, error) { r, err := declaration.ParseResource(raw) diff --git a/go.mod b/go.mod index c100297..785b51d 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/novox/mesh-host go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025204730-0be0ffd6159e + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026105311-175a5567b8bd github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 golang.org/x/crypto v0.57.0 diff --git a/go.sum b/go.sum index f843500..7ce13b1 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025204213-f63fa86c7456 h1:He2o9vSK git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025204213-f63fa86c7456/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025204730-0be0ffd6159e h1:k/1NjjW5HuuuHW2ldyWTvjp+1u3fN0RO18Q57Iup6Dk= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025204730-0be0ffd6159e/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026105311-175a5567b8bd h1:ef6y09csev6xY/suFrIFGUXbgvDA29kummFtHcIb3B8= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026105311-175a5567b8bd/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/internal/link/files.go b/internal/link/files.go index 91e6959..b4a251c 100644 --- a/internal/link/files.go +++ b/internal/link/files.go @@ -10,6 +10,7 @@ import ( "time" "git.novox.be/novox/mesh-sdk/go/declaration" + "git.novox.be/novox/mesh-sdk/go/delivery" "github.com/nats-io/nats.go" ) @@ -92,7 +93,32 @@ func (b OverNATS) FileObject(ctx context.Context, name string, limit int64) ([]b if err != nil { return nil, err } - stream := declaration.FileStreamPrefix + bucket + return objectByName(ctx, js, bucket, declaration.FileStreamPrefix+bucket, name, limit) +} + +// GrantsObject reads one object of the grants part's bucket of sealed user lists by its name (novox/hq ADR 0547 clause 7), +// with the same subjects and nothing more as FileObject: the bus node is granted exactly those (mesh-sdk +// buscompose.GrantsUserListFollows). +func (b OverNATS) GrantsObject(ctx context.Context, name string, limit int64) ([]byte, error) { + js := b.JS + wait := 10 * time.Second + if at, ok := ctx.Deadline(); ok { + if wait = time.Until(at); wait <= 0 { + return nil, context.DeadlineExceeded + } + } + if b.Conn != nil { + bounded, err := b.Conn.JetStream(nats.MaxWait(wait)) + if err != nil { + return nil, err + } + js = bounded + } + return objectByName(ctx, js, delivery.GrantsUserListObjects, delivery.GrantsUserListObjectStream, name, limit) +} + +// objectByName is one object of bucket (its stream stream) by its name, no larger than limit, verified against its digest. +func objectByName(ctx context.Context, js nats.JetStreamContext, bucket, stream, name string, limit int64) ([]byte, error) { meta, err := js.GetLastMsg(stream, "$O."+bucket+".M."+base64.URLEncoding.EncodeToString([]byte(name)), nats.DirectGet(), nats.Context(ctx)) if err != nil { @@ -109,7 +135,7 @@ func (b OverNATS) FileObject(ctx context.Context, name string, limit int64) ([]b return nil, fmt.Errorf("the object %s was deleted from the bucket %s", name, bucket) } if info.Size > uint64(limit) { - return nil, fmt.Errorf("the object is larger than the %d bytes the declaration names", limit) + return nil, fmt.Errorf("the object is larger than the %d bytes allowed", limit) } if info.Size == 0 { return []byte{}, nil @@ -142,6 +168,28 @@ func (l *natsLink) FileObject(ctx context.Context, name string, limit int64) ([] return OverNATS{Conn: l.conn, JS: l.js, Node: l.node}.FileObject(ctx, name, limit) } +func (l *natsLink) GrantsObject(ctx context.Context, name string, limit int64) ([]byte, error) { + return OverNATS{Conn: l.conn, JS: l.js, Node: l.node}.GrantsObject(ctx, name, limit) +} + +// GrantsObjects is a link that can read an object of the grants part's bucket of sealed user lists. +type GrantsObjects interface { + GrantsObject(ctx context.Context, name string, limit int64) ([]byte, error) +} + +// GrantsObject reads an object of the grants part's bucket by its name on the link open now. +func (q *Session) GrantsObject(ctx context.Context, name string, limit int64) ([]byte, error) { + q.init() + q.mu.Lock() + bus := q.bus + q.mu.Unlock() + g, ok := bus.(GrantsObjects) + if bus == nil || !ok { + return nil, ErrNoLink + } + return g.GrantsObject(ctx, name, limit) +} + var ( _ FilesBus = OverNATS{} _ FilesBus = (*natsLink)(nil) diff --git a/internal/userlist/userlist.go b/internal/userlist/userlist.go index 7992a6e..7653ac2 100644 --- a/internal/userlist/userlist.go +++ b/internal/userlist/userlist.go @@ -15,6 +15,13 @@ // older list, and when asked (Again). // // It reads no want and writes none, moves no fingerprint, and touches nothing of the bus module but its user list. +// +// **From take-over it follows the grants part's list instead** (ADR 0547 clauses 7 and 8, issue 1945). On the node whose +// compositions carry the list, it also watches the grants part's one key `user-list.sealed`. While that key says +// FollowGrants, the list is the object the key names, read by its name, opened with this node's sealing key and checked +// against the key's digest, and written to the same file (mode 0600) only when the file's digest differs; otherwise the +// controller's compositions are the source, as before. Never both. The have says which it follows and this node's key, +// which the grants part seals to. package userlist import ( @@ -31,6 +38,7 @@ import ( "time" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/seal" "github.com/novox/mesh-host/internal/agent" ) @@ -59,8 +67,15 @@ type Env struct { Write func(ctx context.Context, module string, resource json.RawMessage, body []byte) (digest string, changed bool, err error) // Report says what was written, in the module's have. Report func(ctx context.Context, module string, w Written) - Now func() time.Time - Say func(string) + // Grants watches the grants part's state (its key delivery.GrantsUserListKey); nil follows the controller's alone. + Grants agent.Watcher + // Fetch reads an object of the grants part's bucket by its name; Open opens a box sealed to this node's key; Key is + // this node's sealing key's public half (SPKI PEM), said in the have so the grants part seals to it ("" unknown). + Fetch func(ctx context.Context, name string) ([]byte, error) + Open func(seal.Box) ([]byte, error) + Key func() string + Now func() time.Time + Say func(string) } type source struct { @@ -83,6 +98,11 @@ type Follower struct { state Written mod string said map[string]bool + // grants is the grants part's key as last heard (nil when none), and following the source last taken: FollowGrants + // or FollowController. + grants *delivery.GrantsUserList + grantsRev uint64 + following string } // New is a follower for a node. @@ -120,6 +140,7 @@ func (f *Follower) Run(ctx context.Context) error { return fmt.Errorf("the user list cannot be followed: %w", err) } caught := false + var grants <-chan agent.Update for { again := false select { @@ -136,13 +157,81 @@ func (f *Follower) Run(ctx context.Context) error { } else { f.take(u) } + case u, ok := <-grants: + if !ok { + // Its watch ended (the link, or a grant the bus took away): heard again at the next event. + grants = nil + f.sayOnce("the grants part's user list is not followed until its key can be watched again") + continue + } + if u.Caught { + continue // nothing changed by the watch catching up + } + f.takeGrants(u) } if caught { f.settle(ctx, again) + // The grants part's key is watched on the node whose compositions carry the list: the bus node alone is + // granted it (mesh-sdk buscompose.GrantsUserListFollows). + if grants == nil && f.env.Grants != nil && f.busNode() { + ch, err := f.env.Grants.Watch(ctx, delivery.GrantsUserListKey) + if err != nil { + f.sayOnce("the grants part's user list cannot be watched yet, so the controller's is followed: " + err.Error()) + } else { + grants = ch + } + } } } } +// busNode says some composition of this node carries the bus's user list. +func (f *Follower) busNode() bool { + f.mu.Lock() + defer f.mu.Unlock() + return f.cur.key != "" || len(f.srcs) > 0 +} + +// takeGrants keeps the grants part's key as heard; a value it cannot read is said and leaves the controller's list +// followed (no list is followed that the node cannot check). +func (f *Follower) takeGrants(u agent.Update) { + f.mu.Lock() + defer f.mu.Unlock() + f.grantsRev = u.Revision + if u.Deleted { + f.grants = nil + return + } + g, err := delivery.DecodeGrantsUserList(u.Value) + if err != nil { + f.grants = nil + f.sayOnceLocked("the grants part's user-list key cannot be read, so the controller's list is followed: " + err.Error()) + return + } + f.grants = &g +} + +// Following is which list the follower takes now: the grants part's while its key says so, the controller's otherwise. +func (f *Follower) Following() string { + f.mu.Lock() + defer f.mu.Unlock() + return f.followingLocked() +} + +func (f *Follower) followingLocked() string { + if f.grants != nil && f.grants.Follow == delivery.FollowGrants { + return delivery.FollowGrants + } + return delivery.FollowController +} + +func (f *Follower) key() string { + if f.env.Key == nil { + return "" + } + return f.env.Key() +} + // take keeps a composition key's change when it carries the bus's user list, and forgets a key deleted. func (f *Follower) take(u agent.Update) { node, module, _, isNode, err := delivery.ParseComposedKey(u.Key) @@ -192,6 +281,17 @@ func (f *Follower) take(u agent.Update) { // A source never goes back: a key deleted, or a newer one refused, leaves the list written as it is, since an older // list would take away accounts a newer one gave. func (f *Follower) settle(ctx context.Context, again bool) { + f.mu.Lock() + follow := f.followingLocked() + if follow != f.following { + // The switch turned: the other source is written, though neither changed. + again = true + } + f.mu.Unlock() + if follow == delivery.FollowGrants { + f.settleGrants(ctx, again) + return + } f.mu.Lock() newest := f.cur for _, s := range f.srcs { @@ -207,7 +307,8 @@ func (f *Follower) settle(ctx context.Context, again bool) { at := fmt.Sprintf("%s@%d", newest.key, newest.rev) f.mu.Unlock() - w := Written{Source: newest.key, Revision: newest.rev, Wanted: newest.wanted, At: f.env.Now().UTC()} + w := Written{Source: newest.key, Revision: newest.rev, Wanted: newest.wanted, At: f.env.Now().UTC(), + Follows: delivery.FollowController, Key: f.key()} var changed bool body, err := f.env.Resolve(ctx, newest.resource) if err == nil { @@ -230,6 +331,7 @@ func (f *Follower) settle(ctx context.Context, again bool) { prev := f.state f.mod, f.state = newest.module, w f.written = err == nil + f.following = delivery.FollowController f.mu.Unlock() if err != nil { f.sayOnce(fmt.Sprintf("the bus's user list of %s could not be written: %v", at, err)) @@ -238,12 +340,108 @@ func (f *Follower) settle(ctx context.Context, again bool) { at, w.Users, short(w.Digest))) } // Reported when what it says changed: a look again that found the file as it was says nothing new. - if f.env.Report != nil && (changed || prev.Source != w.Source || prev.Revision != w.Revision || - prev.Digest != w.Digest || prev.Error != w.Error) { + if f.env.Report != nil && (changed || reportable(prev, w)) { f.env.Report(ctx, newest.module, w) } } +// reportable says a have's user list says something new: a look again that found the file as it was says nothing. +func reportable(prev, w Written) bool { + return prev.Source != w.Source || prev.Revision != w.Revision || prev.Digest != w.Digest || prev.Error != w.Error || + prev.Follows != w.Follows || prev.Key != w.Key +} + +// settleGrants writes the grants part's list, the object its key names, into the file the controller's composition of +// the bus module declares, made 0600 (ADR 0547 clause 7): when the key moved, when the switch turned, or when asked. +func (f *Follower) settleGrants(ctx context.Context, again bool) { + f.mu.Lock() + g, rev := *f.grants, f.grantsRev + file := f.cur + for _, s := range f.srcs { + if s.rev > file.rev { + file = s + } + } + if !again && f.following == delivery.FollowGrants && f.state.Revision == rev && f.written { + f.mu.Unlock() + return + } + f.mu.Unlock() + + w := Written{Source: delivery.GrantsUserListSource, Revision: rev, Wanted: g.Digest, At: f.env.Now().UTC(), + Follows: delivery.FollowGrants, Key: f.key()} + var err error + var changed bool + resource, rerr := mode0600(file.resource) + switch { + case file.key == "" || rerr != nil: + err = fmt.Errorf("no composition of this node declares the bus's user-list file, so the grants part's list has nowhere to be written") + case g.Digest == "": + err = fmt.Errorf("the grants part's key names no list yet; the list on the machine is kept") + case f.env.Fetch == nil || f.env.Open == nil: + err = fmt.Errorf("this engine cannot read or open the grants part's list") + case g.SealedTo != "" && w.Key != "" && g.SealedTo != delivery.SealingKeyID(w.Key): + err = fmt.Errorf("the grants part's list is sealed to another key than this node's (%s); the list on the machine is kept until it seals to this one", g.SealedTo) + } + var body []byte + if err == nil { + var object []byte + if object, err = f.env.Fetch(ctx, g.Object); err == nil { + body, err = delivery.OpenUserList(object, g.Digest, f.env.Open) + } + if err != nil { + err = fmt.Errorf("the grants part's list %s could not be had: %w", short(g.Digest), err) + } + } + if err == nil { + w.Users = Users(body) + if err = f.guard(resource, body); err != nil { + if disk, derr := f.env.OnDisk(resource); derr == nil && disk != nil { + w.Digest = Digest(disk) + } + } + } + if err == nil { + w.Digest, changed, err = f.env.Write(ctx, file.module, resource, body) + } else if w.Digest == "" && file.key != "" && f.env.OnDisk != nil { + if disk, derr := f.env.OnDisk(resource); derr == nil && disk != nil { + w.Digest = Digest(disk) + } + } + if err != nil { + w.Error = err.Error() + } + f.mu.Lock() + prev := f.state + f.mod, f.state = file.module, w + f.written = err == nil + f.following = delivery.FollowGrants + f.mu.Unlock() + at := fmt.Sprintf("%s@%d", delivery.GrantsUserListSource, rev) + if err != nil { + f.sayOnce(fmt.Sprintf("the bus's user list of %s could not be written: %v", at, err)) + } else if changed { + f.env.Say(fmt.Sprintf("the bus's user list from the grants part (%s, revision %d) is written alone (%d users, %s); the bus reloads it itself", + at, g.Revision, w.Users, short(w.Digest))) + } + if f.env.Report != nil && file.module != "" && (changed || reportable(prev, w)) { + f.env.Report(ctx, file.module, w) + } +} + +// mode0600 is a user-list resource made mode 0600, whatever its composition said (ADR 0547 clause 7). +func mode0600(raw json.RawMessage) (json.RawMessage, error) { + if len(raw) == 0 { + return nil, fmt.Errorf("no resource") + } + var r map[string]any + if err := json.Unmarshal(raw, &r); err != nil { + return nil, err + } + r["mode"] = "0600" + return json.Marshal(r) +} + // ControllerUser is the controller's own user in the list: a list without it would lock the controller out of the bus. const ControllerUser = "controller" diff --git a/internal/userlist/userlist_test.go b/internal/userlist/userlist_test.go index d4d52c1..4663e9a 100644 --- a/internal/userlist/userlist_test.go +++ b/internal/userlist/userlist_test.go @@ -11,6 +11,7 @@ import ( "time" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/seal" "github.com/novox/mesh-host/internal/agent" ) @@ -34,6 +35,9 @@ type feed struct { ch chan agent.Update rev uint64 first []agent.Update + // watched is closed once the watch caught up, so a later put is not read before the keys there before it. + watched chan struct{} + once sync.Once } func (f *feed) Watch(context.Context, string) (<-chan agent.Update, error) { @@ -41,6 +45,11 @@ func (f *feed) Watch(context.Context, string) (<-chan agent.Update, error) { f.ch <- u } f.ch <- agent.Update{Caught: true} + f.once.Do(func() { + if f.watched != nil { + close(f.watched) + } + }) return f.ch, nil } @@ -97,6 +106,13 @@ func comp(t *testing.T, module, fp, content string, signer ed25519.PrivateKey, p return k, raw } +// gput is the grants part's key as the bus node hears it at the watch's start (ADR 0547 clauses 7, 8): which list to +// follow, and the list sealed to the node's key; one sealed to another key, or one that does not open to the key's digest. +type gput struct { + follow, content string + otherKey, badDigest bool +} + type put struct { module, fp, content string stranger, deleted bool @@ -118,6 +134,9 @@ func TestTheBusNodeWritesTheNewestSignedUserListAlone(t *testing.T) { wantUsers int wantErr string wantRefused bool + grants []gput + // wantSource and wantFollows are what the last have says it took the list from and follows. + wantSource, wantFollows string }{ {name: "the list there is written once", before: []put{{module: "nats", fp: fpOf("a"), content: two}}, wantWrites: []string{two}, wantUsers: 3}, @@ -153,15 +172,64 @@ func TestTheBusNodeWritesTheNewestSignedUserListAlone(t *testing.T) { onDisk: list("controller", "a", "b", "c", "d", "e"), wantWrites: []string{two}, wantUsers: 3}, {name: "a failed write is said and kept", before: []put{{module: "nats", fp: fpOf("a"), content: two}}, failWrite: true, wantWrites: []string{two}, wantUsers: 3, wantErr: "disk full"}, + // ADR 0547 clauses 7, 8: never both lists; the grants part's from take-over, the controller's otherwise. + {name: "from take-over the grants part's list is written into the same file", + before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grants: []gput{{follow: delivery.FollowGrants, content: three}}, + wantWrites: []string{two, three}, wantUsers: 4, wantSource: delivery.GrantsUserListSource, wantFollows: delivery.FollowGrants}, + {name: "in shadow the grants part's list is not taken", + before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grants: []gput{{follow: delivery.FollowController, content: three}}, + wantWrites: []string{two}, wantUsers: 3, wantSource: "novox.nats." + fpOf("a")[len(delivery.SourcePrefix2):], wantFollows: delivery.FollowController}, + {name: "a grants list that does not open to its digest is refused and the file kept", + before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grants: []gput{{follow: delivery.FollowGrants, content: three, badDigest: true}}, + onDisk: two, wantWrites: []string{two}, wantErr: "could not be had", wantRefused: true, wantSource: delivery.GrantsUserListSource}, + {name: "a grants list sealed to another key is refused and the file kept", + before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grants: []gput{{follow: delivery.FollowGrants, content: three, otherKey: true}}, + onDisk: two, wantWrites: []string{two}, wantErr: "sealed to another key", wantRefused: true, wantSource: delivery.GrantsUserListSource}, } { t.Run(c.name, func(t *testing.T) { - f := &feed{ch: make(chan agent.Update, 64)} + f := &feed{ch: make(chan agent.Update, 64), watched: make(chan struct{})} objects := map[string]string{} var mu sync.Mutex var writes []string var reports []Written reported := make(chan struct{}, 64) - fl := New(Env{Node: node, Composed: f, + gf := &feed{ch: make(chan agent.Update, 64)} + sealed := map[string][]byte{} + nodeKey, err := seal.GenerateKeyPair() + if err != nil { + t.Fatal(err) + } + for i, g := range c.grants { + to := nodeKey.PublicKey + if g.otherKey { + other, _ := seal.GenerateKeyPair() + to = other.PublicKey + } + digest := delivery.UserListDigest([]byte(g.content)) + if g.badDigest { + digest = delivery.UserListDigest([]byte("another list")) + } + object, err := delivery.SealUserList([]byte(g.content), to) + if err != nil { + t.Fatal(err) + } + sealed[delivery.UserListObjectName(digest)] = object + raw, err := delivery.EncodeGrantsUserList(delivery.GrantsUserList{Follow: g.follow, Digest: digest, + Object: delivery.UserListObjectName(digest), SealedTo: delivery.SealingKeyID(to), Node: node, Revision: uint64(i + 1)}) + if err != nil { + t.Fatal(err) + } + gf.first = append(gf.first, agent.Update{Key: delivery.GrantsUserListKey, Value: raw, Revision: uint64(i + 1)}) + } + fl := New(Env{Node: node, Composed: f, Grants: gf, + Fetch: func(_ context.Context, name string) ([]byte, error) { + if o, ok := sealed[name]; ok { + return o, nil + } + return nil, errors.New("no object " + name) + }, + Open: func(b seal.Box) ([]byte, error) { out, err := seal.Open(b, nodeKey.PrivateKey); return []byte(out), err }, + Key: func() string { return nodeKey.PublicKey }, Trusted: func() []ed25519.PublicKey { return []ed25519.PublicKey{controllerPub} }, Resolve: func(_ context.Context, r json.RawMessage) ([]byte, error) { var res struct{ Content, Digest string } @@ -220,6 +288,7 @@ func TestTheBusNodeWritesTheNewestSignedUserListAlone(t *testing.T) { defer cancel() done := make(chan error, 1) go func() { done <- fl.Run(ctx) }() + <-f.watched for _, p := range c.after { f.ch <- update(p) } @@ -252,6 +321,14 @@ func TestTheBusNodeWritesTheNewestSignedUserListAlone(t *testing.T) { t.Fatal("a refused list was not reported") } } + if c.wantSource != "" { + for until := time.Now().Add(5 * time.Second); ; { + if _, st := fl.State(); st.Source == c.wantSource && (c.wantErr == "" || st.Error != "") || time.Now().After(until) { + break + } + time.Sleep(5 * time.Millisecond) + } + } time.Sleep(50 * time.Millisecond) // nothing more is written cancel() <-done @@ -261,6 +338,10 @@ func TestTheBusNodeWritesTheNewestSignedUserListAlone(t *testing.T) { t.Fatalf("written %q, want %q", writes, c.wantWrites) } module, st := fl.State() + if c.wantSource != "" && (st.Source != c.wantSource || st.Key != nodeKey.PublicKey || + (c.wantFollows != "" && st.Follows != c.wantFollows)) { + t.Fatalf("the have says source %q follows %q key %t; want %q %q", st.Source, st.Follows, st.Key == nodeKey.PublicKey, c.wantSource, c.wantFollows) + } if c.wantRefused { if !strings.Contains(st.Error, c.wantErr) || st.Digest != Digest([]byte(c.onDisk)) || st.Users != c.wantUsers { t.Fatalf("refused list's state %+v", st) -- 2.54.0 From 5e2fadce154f27656391d698509e7732b0d1465c Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 12:07:10 +0100 Subject: [PATCH 4/6] Pin mesh-sdk at the group's head 83ee7769 (issue 1945) --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 785b51d..46d0955 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/novox/mesh-host go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026105311-175a5567b8bd + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026110029-83ee7769d8ee github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 golang.org/x/crypto v0.57.0 diff --git a/go.sum b/go.sum index 7ce13b1..87bf518 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025204730-0be0ffd6159e h1:k/1NjjW5 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025204730-0be0ffd6159e/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026105311-175a5567b8bd h1:ef6y09csev6xY/suFrIFGUXbgvDA29kummFtHcIb3B8= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026105311-175a5567b8bd/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026110029-83ee7769d8ee h1:ZdBMkF4WYJi82jb3xYi9Z2lXN/iMTn9r3SFC122ROY4= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026110029-83ee7769d8ee/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= -- 2.54.0 From 61cec66ba8d6ef836a582a517a72331209faa9b5 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 12:08:37 +0100 Subject: [PATCH 5/6] gofmt the follower's test (issue 1945) --- internal/userlist/userlist_test.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/internal/userlist/userlist_test.go b/internal/userlist/userlist_test.go index 4663e9a..9b7c505 100644 --- a/internal/userlist/userlist_test.go +++ b/internal/userlist/userlist_test.go @@ -228,8 +228,11 @@ func TestTheBusNodeWritesTheNewestSignedUserListAlone(t *testing.T) { } return nil, errors.New("no object " + name) }, - Open: func(b seal.Box) ([]byte, error) { out, err := seal.Open(b, nodeKey.PrivateKey); return []byte(out), err }, - Key: func() string { return nodeKey.PublicKey }, + Open: func(b seal.Box) ([]byte, error) { + out, err := seal.Open(b, nodeKey.PrivateKey) + return []byte(out), err + }, + Key: func() string { return nodeKey.PublicKey }, Trusted: func() []ed25519.PublicKey { return []ed25519.PublicKey{controllerPub} }, Resolve: func(_ context.Context, r json.RawMessage) ([]byte, error) { var res struct{ Content, Digest string } -- 2.54.0 From e88ddab2417069b97fe3fcc14a260dad6fd68d31 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 12:54:06 +0100 Subject: [PATCH 6/6] An apply of the bus module waits when the list on the machine cannot be read while the grants part's is followed, so the file is never removed as an orphan (review of the switch, issue 1945) --- cmd/mesh-host/nodeagent.go | 8 +++++++- internal/userlist/userlist.go | 18 ++++++++++++------ internal/userlist/userlist_test.go | 15 ++++++++++++++- 3 files changed, 33 insertions(+), 8 deletions(-) diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index 895403e..d4e69ab 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -451,7 +451,13 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess actx, done := applyContext(session.WithLink(ctx)) // While the grants part's list is followed, the controller's list in the bus module's composition is not // written: the apply keeps the list on the machine (ADR 0547 clause 8, never both). - res := applyComposition(actx, opts, users.Hold(c), others, sched, say) + held, herr := users.Hold(c) + var res agent.Result + if herr != nil { + res = failedWith(herr.Error()) + } else { + res = applyComposition(actx, opts, held, others, sched, say) + } done() // An apply of the bus module may write an older list than the newest composition holds: looked at again. if module, _ := users.State(); module != "" && module == c.Module { diff --git a/internal/userlist/userlist.go b/internal/userlist/userlist.go index 1a9a878..536da9b 100644 --- a/internal/userlist/userlist.go +++ b/internal/userlist/userlist.go @@ -517,18 +517,24 @@ func (f *Follower) settleGrants(ctx context.Context, again bool) { // Hold is c as the apply may write it (C3 of the switch's review): while the grants part's list is followed, the bus // module's composition from the controller carries the controller's list, so its user-list resource holds the list on the -// machine instead (made 0600), or is left out when none can be read. Any other composition, or the controller's list -// followed, is c unchanged. -func (f *Follower) Hold(c delivery.Composed) delivery.Composed { +// machine instead (made 0600), or is left out when the machine holds none. A list on the machine that cannot be read +// is an error: left out, the apply would remove the file as an orphan and the bus would refuse every connection at its +// next reload, so the bus module's apply waits instead. Any other composition, or the controller's list followed, is c +// unchanged. +func (f *Follower) Hold(c delivery.Composed) (delivery.Composed, error) { if f.Following() != delivery.FollowGrants { - return c + return c, nil } out := c out.Resources = nil for _, raw := range c.Resources { if r, _, _, ok := userList([]json.RawMessage{raw}); ok { var held json.RawMessage - if body, err := f.env.OnDisk(r); err == nil && body != nil { + body, err := f.env.OnDisk(r) + if err != nil { + return c, fmt.Errorf("the user list on the machine could not be read, so the apply waits and the file stays: %w", err) + } + if body != nil { var m map[string]any if json.Unmarshal(r, &m) == nil { delete(m, "digest") @@ -544,7 +550,7 @@ func (f *Follower) Hold(c delivery.Composed) delivery.Composed { } out.Resources = append(out.Resources, raw) } - return out + return out, nil } // mode0600 is a user-list resource made mode 0600, whatever its composition said (ADR 0547 clause 7). diff --git a/internal/userlist/userlist_test.go b/internal/userlist/userlist_test.go index fb25d99..94e89b3 100644 --- a/internal/userlist/userlist_test.go +++ b/internal/userlist/userlist_test.go @@ -444,11 +444,24 @@ func TestTheBusNodeWritesTheNewestSignedUserListAlone(t *testing.T) { if err != nil { t.Fatal(err) } - held := fl.Hold(cmp) + held, err := fl.Hold(cmp) + if err != nil { + t.Fatal(err) + } _, _, _, hasList := userList(held.Resources) if want := c.wantFollows == delivery.FollowController; hasList != want || len(held.Resources) < 1 { t.Fatalf("an apply while following %s holds the controller's list: %v", c.wantFollows, hasList) } + // A list on the machine that cannot be read stops the apply; it is never left out to be removed. + if c.wantFollows == delivery.FollowGrants { + readErr := New(Env{Node: node, Composed: f, OnDisk: func(json.RawMessage) ([]byte, error) { + return nil, errors.New("permission denied") + }}) + readErr.grants = &delivery.GrantsUserList{Follow: delivery.FollowGrants} + if _, err := readErr.Hold(cmp); err == nil || !strings.Contains(err.Error(), "could not be read") { + t.Fatalf("an unreadable list on the machine was held as %v", err) + } + } } if c.wantSource != "" && (st.Source != c.wantSource || st.Key != nodeKey.PublicKey || (c.wantFollows != "" && st.Follows != c.wantFollows)) { -- 2.54.0