From a4c0cee25151ec09cb930a91e2decc4617921914 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 31 Oct 2026 09:37:23 +0100 Subject: [PATCH 1/2] Read this node's placement from its node and module keys, or its placement. before the holder converts it, so a write of the holder's is composed only once its node key lands (issue 774, research 072 row 37) --- cmd/mesh-host/nodeagent.go | 50 ++++++++++++----- cmd/mesh-host/nodeagent_integration_test.go | 12 +++- cmd/mesh-host/trust.go | 38 +++++++++++-- go.mod | 2 +- go.sum | 4 +- internal/link/watch.go | 10 +++- internal/selfcompose/selfcompose.go | 41 ++++++++++---- internal/selfcompose/selfcompose_test.go | 61 +++++++++++++++++++-- 8 files changed, 176 insertions(+), 42 deletions(-) diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index 145dc17..d725df7 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -1015,7 +1015,7 @@ type composerHooks struct { } func selfComposer(ctx context.Context, session *link.Session, node string, say func(string), hooks composerHooks) (*selfcompose.Composer, string) { - key, err := delivery.PlacementKey(node) + key, err := delivery.PlacementNodeKey(node) if err != nil { say("this node applies the controller's compositions: " + err.Error()) return nil, err.Error() @@ -1079,31 +1079,55 @@ func composerEnv(session *link.Session, node string, say func(string), hooks com VaultAuthority: hooks.vaultAuthority, Certify: hooks.certify} } +// placementKeysOf is a node's node key and the pattern of the keys that say it is placed: its node key and its legacy +// key `placement.`, read while the holder may not have converted it (research 072 row 37). Its module keys say +// nothing of whether it is placed. +func placementKeysOf(node string) (key, keys string, err error) { + key, err = delivery.PlacementNodeKey(node) + if err != nil { + return "", "", err + } + legacy, _ := delivery.LegacyPlacementKey(node) + return key, link.WatchPatterns(key, legacy), nil +} + +// anyHeld says some key holds a value. +func anyHeld(held map[string]bool) bool { + for _, h := range held { + if h { + return true + } + } + return false +} + // endWhenUnplaced calls stop once this node's placement key is deleted from the seat's placement (after the watch // caught up, or gone already), so the agent begins again and applies the controller's compositions, as a node the // catalogue does not place does from its start: the way back (issue 1126, the seed's revert). A watch that cannot be // made is said; the node then returns at its next start. func endWhenUnplaced(ctx context.Context, session *link.Session, node string, stop func(), say link.Announce) { - key, err := delivery.PlacementKey(node) + key, keys, err := placementKeysOf(node) if err != nil { return } - updates, err := bucketWatch{session, delivery.PlacementBucket}.Watch(ctx, key) + updates, err := bucketWatch{session, delivery.PlacementBucket}.Watch(ctx, keys) if err != nil { say("whether this node's key in the seat's placement state still exists is not followed (" + err.Error() + "): it returns to the controller's compositions at its next start") return } - placed := false + // Its node key or, before the holder converted it, its legacy key: placed while either holds a value (research + // 072 row 37); the holder deletes the legacy key before the node key, so the node key's delete is the last. + held, caught := map[string]bool{}, false for u := range updates { - switch { - case u.Caught && placed: - continue - case u.Caught || u.Deleted || len(u.Value) == 0: + if u.Caught { + caught = true + } else { + held[u.Key] = !u.Deleted && len(u.Value) > 0 + } + if caught && !anyHeld(held) { say("this node's key in the seat's placement state is gone (" + key + "): the node agent begins again, applying the controller's compositions") stop() return - default: - placed = true } } if ctx.Err() == nil { @@ -1115,7 +1139,7 @@ func endWhenUnplaced(ctx context.Context, session *link.Session, node string, st // caught up, or one already there), so the agent begins again and composes its own. A watch that cannot be made is said; // the node then switches at its next start. func endWhenPlaced(ctx context.Context, session *link.Session, node string, stop func(), say link.Announce, note func(string)) { - key, err := delivery.PlacementKey(node) + key, keys, err := placementKeysOf(node) if err != nil { return } @@ -1124,7 +1148,7 @@ func endWhenPlaced(ctx context.Context, session *link.Session, node string, stop // (no timer), says so once in the journal and in the node's own have, where the watchman reads it, and stays on the // controller's compositions until its key is written. A watch the account may not make is said the same way, and the // node stays until its next start. - updates, err := bucketWatch{session, delivery.PlacementBucket}.WatchAwait(ctx, key, func(err error) { + updates, err := bucketWatch{session, delivery.PlacementBucket}.WatchAwait(ctx, keys, func(err error) { why := "this node's key in the seat's placement state is waited for, the state is not on the bus yet (" + err.Error() + "): it stays on the controller's compositions and is watched when the bus says the state was created" say(why) note(why) @@ -1137,7 +1161,7 @@ func endWhenPlaced(ctx context.Context, session *link.Session, node string, stop } for u := range updates { if !u.Caught && !u.Deleted && len(u.Value) > 0 { - say("this node's key in the seat's placement state is written now (" + key + "): the node agent begins again, composing its own modules") + say("this node's key in the seat's placement state is written now (" + u.Key + "): the node agent begins again, composing its own modules") stop() return } diff --git a/cmd/mesh-host/nodeagent_integration_test.go b/cmd/mesh-host/nodeagent_integration_test.go index 852447b..4b36598 100644 --- a/cmd/mesh-host/nodeagent_integration_test.go +++ b/cmd/mesh-host/nodeagent_integration_test.go @@ -498,7 +498,7 @@ func h8back(t *testing.T, b *itBed, placement, resolved, candidates nats.KeyValu strings.Contains(a.Refused, "web: the controller holds no composition") }) ran := b.have("web").Composed - pkey, _ := delivery.PlacementKey(itNode) + pkey, _ := delivery.PlacementNodeKey(itNode) // A pass in flight at the switch (issue 1126, 2026-10-28: the last pass of the own run, failing on its run's end, // marked the next agent's modules failed): a config value changed starts one, and the key goes at once. if _, err := resolved.Put(itNode+".web.greeting", []byte(`{"value":"salut","source":"module"}`)); err != nil { @@ -587,7 +587,10 @@ func h8(t *testing.T, b *itBed, checkBuild bool) (func(), h8state) { modv, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: digest, Repository: "novox/web", Commit: strings.Repeat("1", 40)}) place, _ := delivery.EncodePlacement(delivery.Placement{Node: itNode, Modules: []delivery.PlacedModule{{Module: "web"}}}) rec, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: itNode, Capabilities: []string{"service-manager"}}}) - pkey, _ := delivery.PlacementKey(itNode) + // it1 in one key per module and node (research 072 row 37), it2 in the form before: a node reads both. + pkey, _ := delivery.PlacementNodeKey(itNode) + places, _ := delivery.PlacementValues(delivery.Placement{Node: itNode, Modules: []delivery.PlacedModule{{Module: "web"}}}) + place = places[pkey] hkey, _ := delivery.HaveNodeKey(itNode) // A provider on it2 serving from its own config value: a mail domain, which it1's composition settles. mailer := `{"module":"mailer","version":"1","config":{"domain":{"type":"string","description":"the mail domain","required":true}},` + @@ -598,7 +601,7 @@ func h8(t *testing.T, b *itBed, checkBuild bool) (func(), h8state) { mlmod, _ := delivery.EncodeModule(delivery.Module{FP2: fpIt("b"), Manifest: mdigest, Repository: "novox/mailer", Commit: strings.Repeat("2", 40)}) place2, _ := delivery.EncodePlacement(delivery.Placement{Node: "it2", Modules: []delivery.PlacedModule{{Module: "mailer"}}}) rec2, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: "it2", Capabilities: []string{"service-manager"}}}) - pkey2, _ := delivery.PlacementKey("it2") + pkey2, _ := delivery.LegacyPlacementKey("it2") hkey2, _ := delivery.HaveNodeKey("it2") buildKey, _ := delivery.BuilderBuildKey("web", fp) done, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: digest, @@ -632,6 +635,9 @@ func h8(t *testing.T, b *itBed, checkBuild bool) (func(), h8state) { if b.file("web") != "" { t.Fatal("a node the catalogue does not place composed its own") } + if _, err := placement.Put(itNode+".web", places[itNode+".web"]); err != nil { + t.Fatal(err) + } if _, err := placement.Put(pkey, place); err != nil { t.Fatal(err) } diff --git a/cmd/mesh-host/trust.go b/cmd/mesh-host/trust.go index caac1d7..177918e 100644 --- a/cmd/mesh-host/trust.go +++ b/cmd/mesh-host/trust.go @@ -5,6 +5,7 @@ import ( "crypto/ed25519" "encoding/base64" "encoding/json" + "errors" "fmt" "os" "path/filepath" @@ -239,15 +240,19 @@ func followTrust(ctx context.Context, session *link.Session, trust *meshTrust, s // bus: then the kept declaration is read no more (keptRetired). A node the seat does not place, or whose placement or haves // cannot be read, keeps reading it. func placedAllHave(ctx context.Context, session *link.Session, node string) (bool, string) { - key, err := delivery.PlacementKey(node) + key, err := delivery.PlacementNodeKey(node) if err != nil { return false, err.Error() } - h, err := session.KVHistory(ctx, delivery.PlacementBucket, key) - if err != nil || len(h) == 0 || h[len(h)-1].Deleted { - return false, "its placement is not read" + legacy, _ := delivery.LegacyPlacementKey(node) + // Its node key names every module placed (research 072 row 37); before the holder converted it, its legacy key does. + values := map[string][]byte{} + for _, k := range []string{key, legacy} { + if h, err := session.KVHistory(ctx, delivery.PlacementBucket, k); err == nil && len(h) > 0 && !h[len(h)-1].Deleted { + values[k] = h[len(h)-1].Value + } } - p, err := delivery.DecodePlacement(h[len(h)-1].Value) + p, err := placedModules(node, values) if err != nil { return false, err.Error() } @@ -256,6 +261,29 @@ func placedAllHave(ctx context.Context, session *link.Session, node string) (boo }) } +// placedModules is the modules node's node key names (a placement of their names alone), or its legacy key's placement +// where it has no node key; an error when neither reads. +func placedModules(node string, values map[string][]byte) (delivery.Placement, error) { + key, _ := delivery.PlacementNodeKey(node) + if raw, ok := values[key]; ok { + n, err := delivery.DecodePlacementNode(raw) + if err != nil { + return delivery.Placement{}, err + } + p := delivery.Placement{Node: node} + for _, r := range n.Modules { + p.Modules = append(p.Modules, delivery.PlacedModule{Module: r.Module}) + } + return p, nil + } + legacy, _ := delivery.LegacyPlacementKey(node) + raw, ok := values[legacy] + if !ok { + return delivery.Placement{}, errors.New("its placement is not read") + } + return delivery.DecodePlacement(raw) +} + // allHave says whether has finds, for every module p places, a have that names a fingerprint of the new form, and why // not. A module whose have names only a build of the old form runs inside the node's own composition, and the kept // declaration is the one record of what runs of it: retired then, the node's own composed it as nothing and refused to diff --git a/go.mod b/go.mod index b03b4e6..5f44beb 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/novox/mesh-host go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030235330-dfd5d8295e0a + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031082917-2d1d8def7535 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 golang.org/x/crypto v0.57.0 diff --git a/go.sum b/go.sum index c355c53..df73018 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030235330-dfd5d8295e0a h1:tjbRfUfE6pIylHFXg53SI2Sfgi1+QzWUyaY00xz25z4= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030235330-dfd5d8295e0a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031082917-2d1d8def7535 h1:Bgrm3YfWCxr+FEm03kEP5fDIe9hNbe+j0CDVatGAGKg= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031082917-2d1d8def7535/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/internal/link/watch.go b/internal/link/watch.go index 4d6eeb0..145d5aa 100644 --- a/internal/link/watch.go +++ b/internal/link/watch.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "strings" "time" "github.com/nats-io/nats.go" @@ -23,7 +24,9 @@ type KVUpdate struct { // never polls. type KVBus interface { // KVWatch yields the keys of the bucket matching the pattern now, a Caught update, then every change, until ctx ends - // or the link does (the channel is then closed). A bucket the node's account may not read is an error. + // or the link does (the channel is then closed). A bucket the node's account may not read is an error. The pattern + // may be several, separated by spaces (WatchPatterns), watched as one: a node's placement in either form of its + // keys (research 072 row 37). KVWatch(ctx context.Context, bucket, pattern string) (<-chan KVUpdate, error) } @@ -41,7 +44,7 @@ func (b OverNATS) KVWatch(ctx context.Context, bucket, pattern string) (<-chan K if err != nil { return nil, fmt.Errorf("the bucket %s: %w", bucket, err) } - w, err := kv.Watch(pattern, nats.Context(ctx)) + w, err := kv.WatchFiltered(strings.Fields(pattern), nats.Context(ctx)) if err != nil { return nil, fmt.Errorf("watching %s in %s: %w", pattern, bucket, err) } @@ -296,3 +299,6 @@ func (q *Session) KVWatchAwait(ctx context.Context, bucket, pattern string, wait }() return out, nil } + +// WatchPatterns is several key patterns as one pattern KVWatch takes. +func WatchPatterns(patterns ...string) string { return strings.Join(patterns, " ") } diff --git a/internal/selfcompose/selfcompose.go b/internal/selfcompose/selfcompose.go index 2fe37d3..c83bb8a 100644 --- a/internal/selfcompose/selfcompose.go +++ b/internal/selfcompose/selfcompose.go @@ -98,7 +98,10 @@ type Composer struct { mu sync.Mutex wants map[string]delivery.Want // by module, the node's own under agent.NodeModule placements map[string]delivery.Placement - records map[string]delivery.NodeRecord + // placementKeys is the seat's state placement key by key in either form, each node's placement assembled from it + // (delivery.PlacementKeys; research 072 row 37): a write of the holder's under way is never composed half. + placementKeys *delivery.PlacementKeys + records map[string]delivery.NodeRecord // inOwn is, by module, whether this node's have of it names no fingerprint of the new form (mesh-wireguard, a module // at a build of the old form): the node runs it inside its own composition, and composes it there again (issue 1126). inOwn map[string]bool @@ -215,7 +218,7 @@ func New(env Env) *Composer { if env.Say == nil { env.Say = func(string) {} } - return &Composer{env: env, wants: map[string]delivery.Want{}, placements: map[string]delivery.Placement{}, + return &Composer{env: env, wants: map[string]delivery.Want{}, placements: map[string]delivery.Placement{}, placementKeys: delivery.NewPlacementKeys(), records: map[string]delivery.NodeRecord{}, inOwn: map[string]bool{}, modules: map[string]delivery.Module{}, values: map[string]map[string]any{}, manifests: map[string][]byte{}, archives: map[string][]byte{}, handed: map[string]string{}, sealedOwn: map[string]ownSealed{}, asked: make(chan chan Pass, 8)} @@ -499,7 +502,9 @@ func (c *Composer) entry(bucket, key string) (v any, has, known bool) { w, ok := c.wants[module] return w.FP, ok, true case delivery.PlacementBucket: - node, err := delivery.ParsePlacementKey(key) + // Any key of a node, its node key, a module key or its legacy key, stands for the node's whole placement: a + // module key written before its node key names it changes nothing composed. + node, _, _, err := delivery.ParsePlacementKey(key) if err != nil { return nil, false, true } @@ -551,18 +556,25 @@ func (c *Composer) takeWant(u agent.Update) { } func (c *Composer) takePlacement(u agent.Update) { - node, err := delivery.ParsePlacementKey(u.Key) - if err != nil { + if u.Caught { return } c.mu.Lock() - defer c.mu.Unlock() - p, err := delivery.DecodePlacement(u.Value) - if u.Deleted || err != nil || p.Node != node { - delete(c.placements, node) - return + if c.placementKeys == nil { + c.placementKeys = delivery.NewPlacementKeys() + } + node, err := c.placementKeys.Put(u.Key, u.Value, u.Revision, u.Deleted || len(u.Value) == 0) + if node != "" { + if p, _, ok := c.placementKeys.Of(node); ok { + c.placements[node] = p + } else { + delete(c.placements, node) + } + } + c.mu.Unlock() + if node != "" && err != nil && c.env.Say != nil { + c.env.Say("the seat's placement of " + node + ": " + err.Error()) } - c.placements[node] = p } func (c *Composer) takeRecord(u agent.Update) { @@ -1370,13 +1382,18 @@ func (r reader) Credentials(ctx context.Context, node string, asConsumer, asProv // watch endWhenPlaced keeps). A node that switched never goes back on a failed read, however long it lasts. func Placed(ctx context.Context, get func(ctx context.Context, bucket, key string) ([]byte, error), node string, wait func(try int) time.Duration, say func(string), switched bool) (bool, error) { - key, err := delivery.PlacementKey(node) + key, err := delivery.PlacementNodeKey(node) if err != nil { return false, err } + legacy, _ := delivery.LegacyPlacementKey(node) said := "" for try := 1; ; try++ { + // Its node key, or, before the seat's holder converted it, its legacy key (research 072 row 37). raw, err := get(ctx, delivery.PlacementBucket, key) + if err == nil && len(raw) == 0 { + raw, err = get(ctx, delivery.PlacementBucket, legacy) + } if errors.Is(err, nats.ErrBucketNotFound) { return false, nil // the seat has written no placement state yet: nothing places it, read and known } diff --git a/internal/selfcompose/selfcompose_test.go b/internal/selfcompose/selfcompose_test.go index 0fc00eb..2896afd 100644 --- a/internal/selfcompose/selfcompose_test.go +++ b/internal/selfcompose/selfcompose_test.go @@ -550,7 +550,50 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { // secrets. A key written again with what it held, or a want whose words moved and not its build, makes none. worded, _ := delivery.EncodeWant(delivery.Want{FP: fp, Self: true, Words: "lemurs is held at aaaaaaaaaaaa"}) elsewhere, _ := delivery.EncodeWant(delivery.Want{FP: delivery.SourcePrefix2 + strings.Repeat("e", 64), Self: true}) - for _, row := range []struct { + // Research 072 row 37: the placement converted to one key per module and node moves nothing composed; a write in + // keys makes its pass at its node key, the commit, and not at a module key written before it nor at the mirror. + placeBase := delivery.Placement{Node: "n1", Decided: delivery.NodeDecided{Hub: true, Address: "10.77.0.1"}, + Modules: []delivery.PlacedModule{{Module: "lemurs"}, {Module: "oldie"}}} + placeKeys := delivery.NewPlacementKeys() + keyRows := func(after delivery.Placement, what string) []struct { + name, bucket, key string + value []byte + pass bool + } { + writes, err := placeKeys.Writes("n1", &after) + if err != nil { + t.Fatal(err) + } + var rows []struct { + name, bucket, key string + value []byte + pass bool + } + for i, w := range writes { + _, _ = placeKeys.Put(w.Key, w.Value, uint64(i+1), w.Value == nil) + rows = append(rows, struct { + name, bucket, key string + value []byte + pass bool + }{what + ": " + w.Key, delivery.PlacementBucket, w.Key, w.Value, w.Commit && what != "converted"}) + } + return rows + } + _, _ = placeKeys.Put("placement.n1", place, 1, false) + lemursMoved := placeBase + lemursMoved.Modules = []delivery.PlacedModule{{Module: "lemurs", Values: map[string]any{"greeting": "hoi"}}, {Module: "oldie"}} + var placementRows []struct { + name, bucket, key string + value []byte + pass bool + } + for _, step := range []struct { + what string + p delivery.Placement + }{{"converted", placeBase}, {"lemurs moved", lemursMoved}, {"lemurs back", placeBase}} { + placementRows = append(placementRows, keyRows(step.p, step.what)...) + } + for _, row := range append([]struct { name, bucket, key string value []byte pass bool @@ -562,7 +605,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { {"lemurs's module again", delivery.ModulesBucket, "lemurs", mod, false}, {"lemurs's want at another build", delivery.AssignerWantsBucket, "want.n1.lemurs", elsewhere, true}, {"lemurs's want back at its build", delivery.AssignerWantsBucket, "want.n1.lemurs", want, true}, - } { + }, placementRows...) { before := passes() b.put(t, row.bucket, row.key, row.value) if !row.pass { @@ -690,7 +733,17 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { both, _ := delivery.EncodePlacement(delivery.Placement{Node: "n1", Decided: delivery.NodeDecided{Hub: true, Address: "10.77.0.1"}, Modules: []delivery.PlacedModule{{Module: "lemurs"}, {Module: "oldie"}, {Module: "keeper"}}}) b.put(t, delivery.ModulesBucket, "keeper", kmod) - b.put(t, delivery.PlacementBucket, "placement.n1", both) + // n1 is held in keys per module and node since the rows above, which a legacy key no longer moves. + bothPlaced, _ := delivery.DecodePlacement(both) + bothWrites, err := placeKeys.Writes("n1", &bothPlaced) + if err != nil { + t.Fatal(err) + } + for _, w := range bothWrites { + if w.Value != nil { + b.put(t, delivery.PlacementBucket, w.Key, w.Value) + } + } b.put(t, delivery.AssignerWantsBucket, "want.n1.keeper", want) keeperKey, _ := delivery.ComposedKey("n1", "keeper", fp) var handed string @@ -1227,7 +1280,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { }{ "switched: placed after two failures": {answers: []error{fmt.Errorf("nats: timeout"), fmt.Errorf("nats: timeout")}, value: []byte(`{}`), placed: true, tries: 3, switched: true}, "never switched: a timeout is not placed, said once": {answers: []error{fmt.Errorf("nats: timeout")}, placed: false, tries: 1}, - "no key": {placed: false, tries: 1}, + "no key": {placed: false, tries: 2}, // its node key, then its legacy key (research 072 row 37) "no placement state yet": {answers: []error{fmt.Errorf("the bucket %s: %w", delivery.PlacementBucket, nats.ErrBucketNotFound)}, placed: false, tries: 1, quiet: true}, "switched: failing until the link ends": {answers: []error{fmt.Errorf("jetstream not ready"), fmt.Errorf("jetstream not ready"), fmt.Errorf("jetstream not ready")}, end: true, tries: 3, switched: true}, From 8bdfddab4aab5740b050e972b794061729c32520 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 31 Oct 2026 09:49:14 +0100 Subject: [PATCH 2/2] Hand a module exactly when the vault's own answer judged it handed, so the node never composes a secret the vault leaves to the provisioner (issue 774) --- go.mod | 2 +- go.sum | 4 ++++ internal/selfcompose/selfcompose.go | 20 ++++++++++++++++++++ internal/selfcompose/selfcompose_test.go | 13 ++++++++++++- 4 files changed, 37 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 5f44beb..65a510a 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/novox/mesh-host go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031082917-2d1d8def7535 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 golang.org/x/crypto v0.57.0 diff --git a/go.sum b/go.sum index df73018..1700a57 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,9 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031082917-2d1d8def7535 h1:Bgrm3YfWCxr+FEm03kEP5fDIe9hNbe+j0CDVatGAGKg= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031082917-2d1d8def7535/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031083427-de8e82c06f54 h1:xITIabv8ujmwpaEKBiQpkOx6U/ICLTTzX9WWC3DGOPQ= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031083427-de8e82c06f54/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a h1:DcSvfEmdPBKqCg5OmXRNb269EkkPe7BsEp+RykEhkgE= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/internal/selfcompose/selfcompose.go b/internal/selfcompose/selfcompose.go index c83bb8a..28d4867 100644 --- a/internal/selfcompose/selfcompose.go +++ b/internal/selfcompose/selfcompose.go @@ -19,6 +19,7 @@ import ( "encoding/json" "errors" "fmt" + "maps" "reflect" "slices" "sort" @@ -113,6 +114,10 @@ type Composer struct { // sealedOwn is each own secret's sealed value as first handed, by module/name and generation: a value sealed again // is other bytes, so the one first had is kept while the generation stands, and the composition stays the same. sealedOwn map[string]ownSealed + // ownHanded is, by module, whether the vault judged it handed to the provisioner in its last own answer (its + // `handed`), nil when that answer said nothing of it (a vault older than the field): the composition hands exactly + // those (frombus.HandedReader, issue 774). + ownHanded map[string]bool // certified is the serving key a certificate was last asked for successfully, so it is asked once per key. certified string said string @@ -1279,6 +1284,7 @@ func (r reader) OwnSecrets(ctx context.Context, node string) ([]compose.OwnSecre Generation int `json:"generation"` Sealed json.RawMessage `json:"sealed"` } `json:"own"` + Handed map[string]bool `json:"handed"` } if err := json.Unmarshal(raw, &a); err != nil { return nil, fmt.Errorf("the vault's own answer is not one: %w", err) @@ -1288,6 +1294,7 @@ func (r reader) OwnSecrets(ctx context.Context, node string) ([]compose.OwnSecre } r.c.mu.Lock() defer r.c.mu.Unlock() + r.c.ownHanded = a.Handed out := make([]compose.OwnSecretRow, 0, len(a.Own)) for _, o := range a.Own { // The box first had is kept while its generation stands, so sealing it again moves no composition; the box the @@ -1304,6 +1311,19 @@ func (r reader) OwnSecrets(ctx context.Context, node string) ([]compose.OwnSecre return out, nil } +// Handed is, of this node's modules, which the vault judged handed to the provisioner when it last answered own +// (frombus.HandedReader, issue 774): the node hands exactly those, so the vault and the node judge a module one way. +func (r reader) Handed(_ context.Context, node string) map[string]bool { + if node != r.c.env.Node { + return nil + } + r.c.mu.Lock() + defer r.c.mu.Unlock() + return maps.Clone(r.c.ownHanded) +} + +var _ frombus.HandedReader = reader{} + func (r reader) Credentials(ctx context.Context, node string, asConsumer, asProvider []compose.CredentialKey) ([]compose.Credential, []compose.Issued, error) { if node != r.c.env.Node || r.c.env.Own == nil { return nil, nil, nil diff --git a/internal/selfcompose/selfcompose_test.go b/internal/selfcompose/selfcompose_test.go index 2896afd..29a4a39 100644 --- a/internal/selfcompose/selfcompose_test.go +++ b/internal/selfcompose/selfcompose_test.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "os" + "reflect" "slices" "strconv" "strings" @@ -163,7 +164,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { asks++ sealedAt[fmt.Sprintf(`{"ask":%d}`, asks)] = generation + 1 return json.RawMessage(fmt.Sprintf(`{"node":"n1","key":"sk","own":[{"module":"keeper","name":"password","origin":"made",`+ - `"generation":%d,"sealed":{"ask":%d}}]}`, generation+1, asks)), nil + `"generation":%d,"sealed":{"ask":%d}}],"handed":{"keeper":false}}`, generation+1, asks)), nil } secretChanged := make(chan []byte, 1) env.Hear = func(subject string) (<-chan []byte, func(), error) { @@ -767,6 +768,16 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { if rotated.Key != keeperKey { t.Fatalf("a new generation handed %s, want keeper again", rotated.Key) } + // Issue 774: what the vault judged handed in its own answer is what the node's composition hands, for n1 alone. + b.mu.Lock() + asked := asks + b.mu.Unlock() + if h := (reader{c: c}).Handed(ctx, "n1"); asked == 0 || !reflect.DeepEqual(h, map[string]bool{"keeper": false}) { + t.Errorf("the vault's handed verdicts as the composition reads them, after %d asks: %v", asked, h) + } + if h := (reader{c: c}).Handed(ctx, "n2"); h != nil { + t.Errorf("another node's verdicts: %v", h) + } // Review of L2 (round 2): rotated, then the node-engine restarts. The agent has applied nothing yet, so the box the // node runs comes from keeper's have (Held); the vault seals the value again, other bytes, and the box the have holds