diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index 06ebfd9..8515ea8 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -1153,7 +1153,7 @@ func runLink(ctx context.Context, opts options) error { return identity.LoadSealingKey(identity.SealingKeyPath(opts.state)) }, func(verb string, args any) (json.RawMessage, error) { return queue.AskVault(ctx, verb, args) - }, say, nodeKeyRetryFrom, nodeKeyRetryUpTo) + }, say, nodeKeyRetryFrom, nodeKeyRetryUpTo, vaultKey.heard) // And whether what it runs stays up, looked at on its own clock and said when it changes (novox/hq // ADR 0240) — between applies, which is when a container crash-loops. if judging != nil { diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index 8c47ac4..224dfbc 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -412,6 +412,8 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess } }) }) + // A vault key heard (or heard again changed) looks at the list again, so its hashes are handed (ADR 0547 §9). + vaultKey.following(users.Again) go func() { if err := users.Run(linked); err != nil && linked.Err() == nil { say("the bus's user list is not followed on this link: " + err.Error()) diff --git a/cmd/mesh-host/nodekey.go b/cmd/mesh-host/nodekey.go index f00c2f8..18fe355 100644 --- a/cmd/mesh-host/nodekey.go +++ b/cmd/mesh-host/nodekey.go @@ -2,7 +2,10 @@ package main import ( "context" + "encoding/json" "fmt" + "strings" + "sync" "time" "github.com/novox/mesh-host/internal/identity" @@ -22,11 +25,60 @@ const ( nodeKeyLook = 30 * time.Second ) +// heardKey is the vault's own public key as its answer to node-key gave it (ADR 0547 clause 9, issue 1945): the key the +// bus node hands the loaded list's hashes sealed to. One for the process, as the key stated is; kick is the follower of +// the user list on the link open now, told when the key is first heard or changes. +type heardKey struct { + mu sync.Mutex + pem string + kick func() +} + +// vaultKey is the vault's key this engine heard. +var vaultKey = &heardKey{} + +func (h *heardKey) get() string { + h.mu.Lock() + defer h.mu.Unlock() + return h.pem +} + +// heard keeps the key a node-key answer gave, and tells the follower when it is a first or changed one. An answer +// with no key (a vault before this one) changes nothing. +func (h *heardKey) heard(answer json.RawMessage) { + var a struct { + VaultKey string `json:"vault_key"` + } + if json.Unmarshal(answer, &a) != nil || !strings.Contains(a.VaultKey, "PUBLIC KEY") { + return + } + h.mu.Lock() + changed := a.VaultKey != h.pem + h.pem = a.VaultKey + kick := h.kick + h.mu.Unlock() + if changed && kick != nil { + kick() + } +} + +// following names the follower told when the key changes, and tells it at once when a key is known. +func (h *heardKey) following(kick func()) { + h.mu.Lock() + h.kick = kick + known := h.pem != "" + h.mu.Unlock() + if known && kick != nil { + kick() + } +} + // keepNodeKeyStated states the sealing key load answers to the vault at start, retrying with a growing wait from // retryFrom to retryUpTo while it does not answer, and states it again whenever the key changes. It says one line -// when a key is stated and one when a try fails for another reason than the last, and returns when ctx ends. +// when a key is stated and one when a try fails for another reason than the last, and returns when ctx ends. Each +// answer goes to heard, which takes the vault's own key from it. func keepNodeKeyStated(ctx context.Context, load func() (identity.SealingKey, error), ask vaultAsk, say func(string), - retryFrom, retryUpTo time.Duration) { + retryFrom, retryUpTo time.Duration, heard func(json.RawMessage)) { stated, lastFailure := "", "" wait := retryFrom fail := func(why string) { @@ -45,10 +97,13 @@ func keepNodeKeyStated(ctx context.Context, load func() (identity.SealingKey, er fail(err.Error()) delay = wait case key.Public != stated: - if _, err := ask("node-key", map[string]string{"key": key.Public}); err != nil { + if answer, err := ask("node-key", map[string]string{"key": key.Public}); err != nil { fail(err.Error()) delay = wait } else { + if heard != nil { + heard(answer) + } stated, lastFailure, wait = key.Public, "", retryFrom say(fmt.Sprintf("mesh-host: this machine's sealing key %s is stated to the vault", key.Public)) } diff --git a/cmd/mesh-host/nodekey_test.go b/cmd/mesh-host/nodekey_test.go index 80f85ce..284a326 100644 --- a/cmd/mesh-host/nodekey_test.go +++ b/cmd/mesh-host/nodekey_test.go @@ -71,9 +71,14 @@ func TestTheEngineStatesItsSealingKeyToTheVaultAtStartRetriesAndRestatesAChange( say := func(l string) { lmu.Lock(); lines = append(lines, l); lmu.Unlock() } ctx, cancel := context.WithCancel(context.Background()) defer cancel() - go keepNodeKeyStated(ctx, load, vault.ask, say, time.Millisecond, 5*time.Millisecond) + var hmu sync.Mutex + var answers []string + heard := func(a json.RawMessage) { hmu.Lock(); answers = append(answers, string(a)); hmu.Unlock() } + go keepNodeKeyStated(ctx, load, vault.ask, say, time.Millisecond, 5*time.Millisecond, heard) waitFor(t, "the first key stated after two refusals", func() bool { return len(vault.said()) == 1 }) + // Its answer is handed on, so the vault's own key in it is heard (ADR 0547 clause 9); refusals hand nothing. + waitFor(t, "the answer heard", func() bool { hmu.Lock(); defer hmu.Unlock(); return len(answers) == 1 }) if vault.said()[0] != first.Public { t.Errorf("stated %q, want the engine's own key %q", vault.said()[0], first.Public) } @@ -104,9 +109,30 @@ func TestTheEngineStatesItsSealingKeyToTheVaultAtStartRetriesAndRestatesAChange( } } -// The statement a host makes names this contract (issue 786), one above the vault's box. +// The statement a host makes names this contract (issue 786), one above the vault's box. And the vault's key its +// answer gives is kept, told to the follower once when first heard and again only when changed; an answer without one +// (an older vault) or with no public key changes nothing (ADR 0547 clause 9). func TestTheNodeKeyContractIsThirteen(t *testing.T) { if link.NodeKeyContract != 13 { t.Errorf("NodeKeyContract %d", link.NodeKeyContract) } + h := &heardKey{} + kicks := 0 + h.following(func() { kicks++ }) + h.heard(json.RawMessage(`{"recorded":true}`)) + h.heard(json.RawMessage(`{"recorded":true,"vault_key":"not a key"}`)) + if h.get() != "" || kicks != 0 { + t.Fatalf("an answer with no key was kept: %q, %d kicks", h.get(), kicks) + } + pem := "-----BEGIN PUBLIC KEY-----\nAAAA\n-----END PUBLIC KEY-----\n" + answer, _ := json.Marshal(map[string]any{"recorded": true, "vault_key": pem}) + h.heard(answer) + h.heard(answer) + if h.get() != pem || kicks != 1 { + t.Fatalf("key %q, %d kicks, want the key and one", h.get(), kicks) + } + h.following(func() { kicks += 10 }) + if kicks != 11 { + t.Fatalf("a follower named after the key was heard was not told: %d", kicks) + } } diff --git a/cmd/mesh-host/userlist.go b/cmd/mesh-host/userlist.go index ff5527a..642074e 100644 --- a/cmd/mesh-host/userlist.go +++ b/cmd/mesh-host/userlist.go @@ -42,7 +42,10 @@ func userListFollower(opts options, session *link.Session, composedBucket string Report: func(ctx context.Context, module string, w userlist.Written) { session.NoteUserList(ctx, module, w) }, - Say: announceOr(say), + // The vault's key from its answer to node-key: the loaded list's hashes are handed sealed to it (ADR 0547 + // clause 9). + VaultKey: vaultKey.get, + Say: announceOr(say), }) } diff --git a/go.mod b/go.mod index c100297..f096840 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/novox/mesh-host go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025204730-0be0ffd6159e + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026102838-174789941936 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 golang.org/x/crypto v0.57.0 diff --git a/go.sum b/go.sum index f843500..bb8a904 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025204213-f63fa86c7456 h1:He2o9vSK git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025204213-f63fa86c7456/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025204730-0be0ffd6159e h1:k/1NjjW5HuuuHW2ldyWTvjp+1u3fN0RO18Q57Iup6Dk= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025204730-0be0ffd6159e/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026102838-174789941936 h1:HGBoWimZXYZp+Qn9VCstYDitOP84t9EUQYd1yM9j2eg= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026102838-174789941936/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/internal/userlist/userlist.go b/internal/userlist/userlist.go index 7992a6e..1873ed7 100644 --- a/internal/userlist/userlist.go +++ b/internal/userlist/userlist.go @@ -15,6 +15,11 @@ // older list, and when asked (Again). // // It reads no want and writes none, moves no fingerprint, and touches nothing of the bus module but its user list. +// +// **It hands the loaded list's hashes to the vault** (ADR 0547 clause 9, issue 1945): once the vault's key is known +// (VaultKey), the have of the bus module carries the bcrypt hash of every user of the list on the machine, sealed to +// that key (delivery.AccountsHanded), so the vault adopts the accounts it holds no value for. It is sealed again only +// when the list on the machine or the vault's key changes. package userlist import ( @@ -59,8 +64,11 @@ type Env struct { Write func(ctx context.Context, module string, resource json.RawMessage, body []byte) (digest string, changed bool, err error) // Report says what was written, in the module's have. Report func(ctx context.Context, module string, w Written) - Now func() time.Time - Say func(string) + // VaultKey is the vault's own public key (SPKI PEM) as its answer to node-key gave it, "" while none is known: the + // key the loaded list's hashes are handed sealed to. Nil hands nothing. + VaultKey func() string + Now func() time.Time + Say func(string) } type source struct { @@ -220,14 +228,24 @@ func (f *Follower) settle(ctx context.Context, again bool) { } } } + var loaded []byte if err == nil { w.Digest, changed, err = f.env.Write(ctx, newest.module, newest.resource, body) + if err == nil { + loaded = body + } + } else if body != nil && f.env.OnDisk != nil { + // Refused: the list on the machine stays, and it is the one the bus has loaded. + loaded, _ = f.env.OnDisk(newest.resource) } if err != nil { w.Error = err.Error() } f.mu.Lock() prev := f.state + f.mu.Unlock() + w.Handed = f.hand(loaded, prev.Handed) + f.mu.Lock() f.mod, f.state = newest.module, w f.written = err == nil f.mu.Unlock() @@ -239,11 +257,47 @@ func (f *Follower) settle(ctx context.Context, again bool) { } // Reported when what it says changed: a look again that found the file as it was says nothing new. if f.env.Report != nil && (changed || prev.Source != w.Source || prev.Revision != w.Revision || - prev.Digest != w.Digest || prev.Error != w.Error) { + prev.Digest != w.Digest || prev.Error != w.Error || handedKey(prev.Handed) != handedKey(w.Handed)) { f.env.Report(ctx, newest.module, w) } } +// hand is the handing of the list on the machine to the vault: the one before when it is of the same list and key, a +// new one sealed now otherwise, and none while the vault's key is not known or the list holds no hash. +func (f *Follower) hand(loaded []byte, before *delivery.AccountsHanded) *delivery.AccountsHanded { + if f.env.VaultKey == nil || loaded == nil { + return before + } + key := f.env.VaultKey() + if key == "" { + return before + } + from := Digest(loaded) + if before != nil && before.From == from && before.To == delivery.KeyFingerprint(key) { + return before + } + accounts := delivery.HandedAccountsOf(loaded) + if len(accounts) == 0 { + return nil + } + h, err := delivery.HandAccounts(accounts, key, from, f.env.Now()) + if err != nil { + f.sayOnce(fmt.Sprintf("the loaded list's hashes are not handed to the vault: %v", err)) + return before + } + f.env.Say(fmt.Sprintf("the hashes of the %d users of the bus's loaded list (%s) are handed to the vault, sealed to "+ + "its key %s, in the bus module's have", h.Users, short(from), short(h.To))) + return &h +} + +// handedKey is what a handing says that a report compares: its list and key. +func handedKey(h *delivery.AccountsHanded) string { + if h == nil { + return "" + } + return h.From + " " + h.To +} + // ControllerUser is the controller's own user in the list: a list without it would lock the controller out of the bus. const ControllerUser = "controller" diff --git a/internal/userlist/userlist_test.go b/internal/userlist/userlist_test.go index d4d52c1..55f8080 100644 --- a/internal/userlist/userlist_test.go +++ b/internal/userlist/userlist_test.go @@ -11,6 +11,7 @@ import ( "time" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/seal" "github.com/novox/mesh-host/internal/agent" ) @@ -29,11 +30,13 @@ var _, strangerKey = key() func fpOf(c string) string { return delivery.SourcePrefix2 + strings.Repeat(c, 64) } -// feed is a watch the test writes to: the keys there before the watch, Caught, then each put. +// feed is a watch the test writes to: the keys there before the watch, Caught, then each put. watching is closed once +// the watch began, so a put after it is never handed before the keys there before it. type feed struct { - ch chan agent.Update - rev uint64 - first []agent.Update + ch chan agent.Update + rev uint64 + first []agent.Update + watching chan struct{} } func (f *feed) Watch(context.Context, string) (<-chan agent.Update, error) { @@ -41,9 +44,23 @@ func (f *feed) Watch(context.Context, string) (<-chan agent.Update, error) { f.ch <- u } f.ch <- agent.Update{Caught: true} + close(f.watching) return f.ch, nil } +// hashed is a list whose users carry a hash of bcrypt's form, as the composer writes them. +func hashed(users ...string) string { + var b strings.Builder + b.WriteString("authorization {\n users = [\n") + for _, u := range users { + b.WriteString(` { user: "` + u + `", password: "$2a$11$` + strings.Repeat("h", 53) + `", permissions: {` + "\n") + } + b.WriteString(" ]\n}\n") + return b.String() +} + +var vault, _ = seal.GenerateKeyPair() + func list(users ...string) string { var b strings.Builder b.WriteString("authorization {\n users = [\n") @@ -118,6 +135,10 @@ func TestTheBusNodeWritesTheNewestSignedUserListAlone(t *testing.T) { wantUsers int wantErr string wantRefused bool + // vault gives the follower the vault's key; wantHanded is how many users it then hands from the list on the + // machine (ADR 0547 clause 9). + vault bool + wantHanded int }{ {name: "the list there is written once", before: []put{{module: "nats", fp: fpOf("a"), content: two}}, wantWrites: []string{two}, wantUsers: 3}, @@ -151,11 +172,17 @@ func TestTheBusNodeWritesTheNewestSignedUserListAlone(t *testing.T) { onDisk: five, wantUsers: 2, wantErr: "drops more than half", wantRefused: true}, {name: "a list that drops half is written", before: []put{{module: "nats", fp: fpOf("a"), content: two}}, onDisk: list("controller", "a", "b", "c", "d", "e"), wantWrites: []string{two}, wantUsers: 3}, + {name: "the loaded list's hashes are handed sealed to the vault's key", + before: []put{{module: "nats", fp: fpOf("a"), content: hashed("controller", "node.ace")}}, vault: true, + wantWrites: []string{hashed("controller", "node.ace")}, wantUsers: 2, wantHanded: 2}, + {name: "a refused list hands the list on the machine", + before: []put{{module: "nats", fp: fpOf("a"), content: list("a", "b")}}, vault: true, + onDisk: hashed("controller", "a", "b"), wantUsers: 2, wantErr: "holds no user", wantRefused: true, wantHanded: 3}, {name: "a failed write is said and kept", before: []put{{module: "nats", fp: fpOf("a"), content: two}}, failWrite: true, wantWrites: []string{two}, wantUsers: 3, wantErr: "disk full"}, } { t.Run(c.name, func(t *testing.T) { - f := &feed{ch: make(chan agent.Update, 64)} + f := &feed{ch: make(chan agent.Update, 64), watching: make(chan struct{})} objects := map[string]string{} var mu sync.Mutex var writes []string @@ -197,6 +224,12 @@ func TestTheBusNodeWritesTheNewestSignedUserListAlone(t *testing.T) { reported <- struct{}{} }, Now: func() time.Time { return time.Date(2026, 10, 25, 21, 50, 0, 0, time.UTC) }, + VaultKey: func() string { + if c.vault { + return vault.PublicKey + } + return "" + }, }) update := func(p put) agent.Update { f.rev++ @@ -220,6 +253,7 @@ func TestTheBusNodeWritesTheNewestSignedUserListAlone(t *testing.T) { defer cancel() done := make(chan error, 1) go func() { done <- fl.Run(ctx) }() + <-f.watching for _, p := range c.after { f.ch <- update(p) } @@ -261,6 +295,22 @@ func TestTheBusNodeWritesTheNewestSignedUserListAlone(t *testing.T) { t.Fatalf("written %q, want %q", writes, c.wantWrites) } module, st := fl.State() + switch { + case c.wantHanded == 0 && st.Handed != nil: + t.Fatalf("handed with no vault key or no hash: %+v", st.Handed) + case c.wantHanded > 0: + loaded := c.onDisk + if !c.wantRefused { + loaded = c.wantWrites[len(c.wantWrites)-1] + } + if st.Handed == nil || st.Handed.From != Digest([]byte(loaded)) { + t.Fatalf("handed %+v, want from the list on the machine", st.Handed) + } + opened, err := delivery.OpenHanded(*st.Handed, vault.PublicKey, vault.PrivateKey) + if err != nil || len(opened) != c.wantHanded || opened[0].User != "controller" { + t.Fatalf("opened %+v, %v", opened, err) + } + } if c.wantRefused { if !strings.Contains(st.Error, c.wantErr) || st.Digest != Digest([]byte(c.onDisk)) || st.Users != c.wantUsers { t.Fatalf("refused list's state %+v", st)