diff --git a/go.mod b/go.mod index 0a71ab3..c5812c9 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/novox/mesh-host go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027014310-cadd7b1b2f8f + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027075026-05b678310366 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 golang.org/x/crypto v0.57.0 diff --git a/go.sum b/go.sum index 04ad7e0..ae78f0e 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027012704-062c29c3a22f h1:AKzV973U git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027012704-062c29c3a22f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027014310-cadd7b1b2f8f h1:mimAGERguQmCGYVNdRTrFcZ1dhpU6GWo+Q3EDhHOrVk= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027014310-cadd7b1b2f8f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027075026-05b678310366 h1:B7oQ82ar2McqZ9OrbJfyvG+50Ol6fBUoK3eYE2uTPvo= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027075026-05b678310366/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/internal/selfcompose/selfcompose.go b/internal/selfcompose/selfcompose.go index 12ef2a7..24f190f 100644 --- a/internal/selfcompose/selfcompose.go +++ b/internal/selfcompose/selfcompose.go @@ -71,10 +71,13 @@ type Composer struct { wants map[string]delivery.Want // by module, the node's own under agent.NodeModule placements map[string]delivery.Placement records map[string]delivery.NodeRecord - modules map[string]delivery.Module - values map[string]map[string]any // by module, this node's - manifests map[string][]byte // by digest, read once - handed map[string]string // composed key to the digest handed + // inOwn is, by module, whether this node's have of it names no fingerprint of the new form (mesh-wireguard, a module + // at a build of the old form): the node runs it inside its own composition, and composes it there again (issue 1126). + inOwn map[string]bool + modules map[string]delivery.Module + values map[string]map[string]any // by module, this node's + manifests map[string][]byte // by digest, read once + handed map[string]string // composed key to the digest handed // sealedOwn is each own secret's sealed value as first handed, by module/name and generation: a value sealed again // is other bytes, so the one first had is kept while the generation stands, and the composition stays the same. sealedOwn map[string]ownSealed @@ -90,7 +93,7 @@ func New(env Env) *Composer { env.Say = func(string) {} } return &Composer{env: env, wants: map[string]delivery.Want{}, placements: map[string]delivery.Placement{}, - records: map[string]delivery.NodeRecord{}, modules: map[string]delivery.Module{}, values: map[string]map[string]any{}, + records: map[string]delivery.NodeRecord{}, inOwn: map[string]bool{}, modules: map[string]delivery.Module{}, values: map[string]map[string]any{}, manifests: map[string][]byte{}, handed: map[string]string{}, sealedOwn: map[string]ownSealed{}} } @@ -105,16 +108,19 @@ type ownSealed struct { type input struct { bucket, pattern string take func(c *Composer, u agent.Update) + // changed, when set, takes the update in take's place and says whether it changed anything composed. + changed func(c *Composer, u agent.Update) bool } func (c *Composer) inputs() []input { node := c.env.Node return []input{ - {delivery.AssignerWantsBucket, "want." + node + ".>", (*Composer).takeWant}, - {delivery.PlacementBucket, ">", (*Composer).takePlacement}, - {delivery.HaveBucket, "*." + delivery.HaveNodeWord, (*Composer).takeRecord}, - {delivery.ModulesBucket, ">", (*Composer).takeModule}, - {ResolvedBucket, node + ".>", (*Composer).takeValue}, + {delivery.AssignerWantsBucket, "want." + node + ".>", (*Composer).takeWant, nil}, + {delivery.PlacementBucket, ">", (*Composer).takePlacement, nil}, + // Every node's record, and this node's haves: what it runs inside its own composition. + {delivery.HaveBucket, ">", nil, (*Composer).takeHave}, + {delivery.ModulesBucket, ">", (*Composer).takeModule, nil}, + {ResolvedBucket, node + ".>", (*Composer).takeValue, nil}, } } @@ -193,6 +199,8 @@ func (c *Composer) Watch(ctx context.Context, _ string) (<-chan agent.Update, er dirty = true // a secret changed: its new generation is asked of the vault at the compose case t.u.Caught: caught[t.in] = true + case ins[t.in].changed != nil: + dirty = ins[t.in].changed(c, t.u) || dirty default: ins[t.in].take(c, t.u) dirty = true @@ -302,6 +310,56 @@ func (c *Composer) takeRecord(u agent.Update) { c.records[node] = *h.Record } +// takeHave takes a have key: a node's record (always composed again), or one of this node's modules' haves, which +// changes what is composed only when whether it runs inside the node's own composition changes. +func (c *Composer) takeHave(u agent.Update) bool { + if isNodeKey(u.Key) { + c.takeRecord(u) + return true + } + return c.takeModuleHave(u) +} + +// takeModuleHave takes one of this node's modules' haves, and says whether it changed which it runs inside its own +// composition: one whose have names no fingerprint of the new form and is not removed (nats on novox: a build-... +// id; mesh-wireguard: none). Read from the haves, which outlast a restart, never from what the agent has applied +// this run: a fresh agent composes before it applies anything (review of the fold, round 1). +func (c *Composer) takeModuleHave(u agent.Update) bool { + node, module, isNode, err := delivery.ParseHaveKey(u.Key) + if err != nil || isNode || node != c.env.Node { + return false + } + in := false + if !u.Deleted { + if h, err := delivery.DecodeHave(u.Value); err == nil { + in = h.Phase != delivery.PhaseRemoved && !strings.HasPrefix(h.FP, delivery.SourcePrefix2) + } + } + c.mu.Lock() + defer c.mu.Unlock() + if c.inOwn[module] == in { + return false + } + if in { + c.inOwn[module] = true + } else { + delete(c.inOwn, module) + } + return true +} + +// inOwnOf is, of the modules p places, those the node runs inside its own composition (c.inOwn), sorted; c.mu held. +func (c *Composer) inOwnOf(p delivery.Placement) []string { + var out []string + for _, m := range p.Modules { + if c.inOwn[m.Module] { + out = append(out, m.Module) + } + } + sort.Strings(out) + return out +} + func (c *Composer) takeModule(u agent.Update) { name, err := delivery.ParseModulesKey(u.Key) if err != nil { @@ -359,6 +417,9 @@ func (c *Composer) compose(ctx context.Context) []agent.Update { } } _, nodeWanted := c.wants[agent.NodeModule] + // What it runs inside its own composition and no want names at a build of the new form stays there, as the + // controller's composed writer folds it (issue 1126); compose folds none a want names. + inOwn := c.inOwnOf(c.placements[c.env.Node]) r := reader{c: c} c.mu.Unlock() sort.Strings(behind) @@ -373,7 +434,7 @@ func (c *Composer) compose(ctx context.Context) []agent.Update { kept[module] = digest } - compositions, _, err := frombus.Compose(ctx, r, c.env.Mesh, frombus.Scope{Node: c.env.Node, Kept: kept}, fps) + compositions, _, err := frombus.Compose(ctx, r, c.env.Mesh, frombus.Scope{Node: c.env.Node, Kept: kept, InOwn: inOwn}, fps) if err != nil { c.refused(fmt.Sprintf("%v", err)) c.sayOnce(fmt.Sprintf("the node cannot compose its modules, and keeps running what it runs: %v", err)) diff --git a/internal/selfcompose/selfcompose_test.go b/internal/selfcompose/selfcompose_test.go index ba3a246..3175f53 100644 --- a/internal/selfcompose/selfcompose_test.go +++ b/internal/selfcompose/selfcompose_test.go @@ -112,21 +112,33 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { if err != nil { t.Fatal(err) } + // oldie is placed and no want names it (a module at a build of the old form, nats on novox): the node runs it inside + // its own composition, and composes it there again, even before the agent has applied anything (Running empty). + oldie := `{"module":"oldie","version":"1","capabilities":["systemd"],"resources":[{"id":"service","type":"service","unit":"oldie.service","state":"running","boot":"enabled"}]}` + odigest := delivery.ManifestDigest([]byte(oldie)) + oman, _ := delivery.EncodeManifest(delivery.Manifest{Module: "oldie", JSON: oldie}) + okey, _ := delivery.ManifestsKey(odigest) + b.kv[delivery.ManifestsBucket+"/"+okey] = oman + omod, _ := delivery.EncodeModule(delivery.Module{FP2: delivery.SourcePrefix2 + strings.Repeat("c", 64), Manifest: odigest, Repository: "novox/o", Commit: "c"}) place, _ := delivery.EncodePlacement(delivery.Placement{Node: "n1", Decided: delivery.NodeDecided{Hub: true, Address: "10.77.0.1"}, - Modules: []delivery.PlacedModule{{Module: "lemurs"}}}) + Modules: []delivery.PlacedModule{{Module: "lemurs"}, {Module: "oldie"}}}) have, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: "n1", TunnelKey: "k", SealingKey: "sk", Capabilities: []string{"systemd"}}}) mod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: digest, Repository: "novox/x", Commit: "c"}) want, _ := delivery.EncodeWant(delivery.Want{FP: fp, Self: true}) nodeWant, _ := delivery.EncodeNodeWant(delivery.NodeWant{Self: true}) b.put(t, delivery.PlacementBucket, "placement.n1", place) b.put(t, delivery.HaveBucket, "n1.node", have) + oldHave, _ := delivery.EncodeHave(delivery.Have{Build: "build-1", Phase: delivery.PhaseRunning, Health: delivery.HealthOK}) + b.put(t, delivery.HaveBucket, "n1.oldie", oldHave) b.put(t, delivery.ModulesBucket, "lemurs", mod) + b.put(t, delivery.ModulesBucket, "oldie", omod) b.put(t, delivery.AssignerWantsBucket, "want.n1.lemurs", want) b.put(t, delivery.AssignerWantsBucket, "want.n1.node", nodeWant) for _, bucket := range []string{delivery.AssignerWantsBucket, delivery.PlacementBucket, delivery.HaveBucket, delivery.ModulesBucket, ResolvedBucket} { b.ch(bucket) <- agent.Update{Caught: true} } got := map[string]bool{} + var own delivery.Composed for { u := next(t, out) if u.Caught { @@ -139,8 +151,18 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { if c.Node != "n1" { t.Errorf("%s composed for %q", u.Key, c.Node) } + if c.Module == "" { + own = c + } got[u.Key] = true } + var ownText []string + for _, r := range own.Resources { + ownText = append(ownText, string(r)) + } + if !strings.Contains(strings.Join(ownText, " "), `"id":"oldie.service"`) { + t.Errorf("the node's own composition, composed before anything was applied, lacks oldie: %v", ownText) + } lemursKey, _ := delivery.ComposedKey("n1", "lemurs", fp) nodeKey, _ := delivery.ComposedNodeKey("n1") if !got[lemursKey] || !got[nodeKey] || len(got) != 2 { @@ -196,7 +218,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { b.mu.Unlock() kmod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: kdigest, Repository: "novox/k", Commit: "c"}) both, _ := delivery.EncodePlacement(delivery.Placement{Node: "n1", Decided: delivery.NodeDecided{Hub: true, Address: "10.77.0.1"}, - Modules: []delivery.PlacedModule{{Module: "lemurs"}, {Module: "keeper"}}}) + Modules: []delivery.PlacedModule{{Module: "lemurs"}, {Module: "oldie"}, {Module: "keeper"}}}) b.put(t, delivery.ModulesBucket, "keeper", kmod) b.put(t, delivery.PlacementBucket, "placement.n1", both) b.put(t, delivery.AssignerWantsBucket, "want.n1.keeper", want) @@ -444,6 +466,17 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { _, ok := a["drops"] return !ok && len(a) == 2 }) + + // Issue 1126, the seed's review: a module n1 runs inside its own composition (nats, at a build of the old form) is + // composed there again, not at the trunk's build as one added; lemurs at the build it runs, keeper (added) at the + // trunk's. + inOwnRunning := []delivery.Composed{{Node: "n1", Module: "lemurs", FP: fp}} + other := "src2:" + strings.Repeat("9", 64) + gotFps := shadowBuilds(inOwnRunning, delivery.Placement{Node: "n1", Modules: []delivery.PlacedModule{{Module: "lemurs"}, {Module: "nats"}, {Module: "keeper"}}}, + map[string]delivery.Module{"lemurs": {FP2: other}, "nats": {FP2: other}, "keeper": {FP2: other}}, []string{"nats"}) + if fmt.Sprint(gotFps) != fmt.Sprint(map[string]string{"lemurs": fp, "keeper": other}) { + t.Errorf("shadow builds %v; want lemurs at what runs, keeper at the trunk's, nats at none (folded)", gotFps) + } } // patternBus is a bus whose every watch gets the updates of its bucket matching its pattern, as the bus's own watches diff --git a/internal/selfcompose/shadow.go b/internal/selfcompose/shadow.go index 2bbdb1d..e03cc51 100644 --- a/internal/selfcompose/shadow.go +++ b/internal/selfcompose/shadow.go @@ -89,13 +89,13 @@ func (s *Shadow) Run(ctx context.Context) error { c := s.c node := c.env.Node ins := []input{ - {delivery.PlacementCandidatesBucket, ">", nil}, - {delivery.PlacementBucket, ">", (*Composer).takePlacement}, - {delivery.HaveBucket, "*." + delivery.HaveNodeWord, nil}, - {delivery.ModulesBucket, ">", (*Composer).takeModule}, - {ResolvedBucket, node + ".>", (*Composer).takeValue}, + {delivery.PlacementCandidatesBucket, ">", nil, nil}, + {delivery.PlacementBucket, ">", (*Composer).takePlacement, nil}, + {delivery.HaveBucket, "*." + delivery.HaveNodeWord, nil, nil}, + {delivery.ModulesBucket, ">", (*Composer).takeModule, nil}, + {ResolvedBucket, node + ".>", (*Composer).takeValue, nil}, // What this node runs moved: an apply ends in its module's have. - {delivery.HaveBucket, node + ".*", nil}, + {delivery.HaveBucket, node + ".*", nil, nil}, } type tagged struct { in int @@ -140,6 +140,7 @@ func (s *Shadow) Run(ctx context.Context) error { case t.in == 2: dirty = s.takeRecord(t.u) || dirty case t.in == 5: + c.takeModuleHave(t.u) dirty = !isNodeKey(t.u.Key) || dirty default: ins[t.in].take(c, t.u) @@ -305,14 +306,6 @@ func (s *Shadow) answerGroup(ctx context.Context, group string) (delivery.Candid return a, true } - // Each module at the build it runs; one the candidate adds at the trunk's build. - fps := map[string]string{} - kept := map[string]string{} - for _, r := range running { - if r.Module != "" && r.FP != "" { - fps[r.Module] = r.FP - } - } placement := given[node] if placement == nil { c.mu.Lock() @@ -320,13 +313,12 @@ func (s *Shadow) answerGroup(ctx context.Context, group string) (delivery.Candid c.mu.Unlock() placement = &p } - for _, m := range placement.Modules { - if _, runs := fps[m.Module]; !runs { - if mod, ok := modules[m.Module]; ok && mod.FP2 != "" { - fps[m.Module] = mod.FP2 - } - } - } + // What it runs inside its own composition, of what the candidate places (the seed's trunk places nothing). + c.mu.Lock() + inOwn := c.inOwnOf(*placement) + c.mu.Unlock() + fps := shadowBuilds(running, *placement, modules, inOwn) + kept := map[string]string{} for module, fp := range fps { if mod, ok := modules[module]; ok && mod.FP2 == fp { continue @@ -347,10 +339,36 @@ func (s *Shadow) answerGroup(ctx context.Context, group string) (delivery.Candid c.mu.Lock() c.wants = held c.mu.Unlock() - a := frombus.Answer(ctx, reader{c: c}, c.env.Mesh, frombus.Scope{Node: node, Kept: kept, Placements: given}, fps, running, head, rev) + a := frombus.Answer(ctx, reader{c: c}, c.env.Mesh, frombus.Scope{Node: node, Kept: kept, Placements: given, InOwn: inOwn}, fps, running, head, rev) return a, true } +// shadowBuilds is the build each module placed is composed at in shadow: the one it runs, and for one the candidate +// adds, the trunk's. A module it runs inside its own composition (inOwn, from its haves: mesh-wireguard, one at a build +// of the old form) is given none, so it is folded there again, as the node's own composer folds it, rather than +// composed at the trunk's build as if the candidate added it (issue 1126: nats and photos have a trunk build that no +// want names). +func shadowBuilds(running []delivery.Composed, placement delivery.Placement, modules map[string]delivery.Module, inOwn []string) map[string]string { + fps := map[string]string{} + for _, r := range running { + if r.Module != "" && r.FP != "" { + fps[r.Module] = r.FP + } + } + folded := map[string]bool{} + for _, m := range inOwn { + folded[m] = true + } + for _, m := range placement.Modules { + if _, runs := fps[m.Module]; !runs && !folded[m.Module] { + if mod, ok := modules[m.Module]; ok && mod.FP2 != "" { + fps[m.Module] = mod.FP2 + } + } + } + return fps +} + func sameAnswer(a, b delivery.CandidateAnswer) bool { x, errA := delivery.EncodeHaveNode(delivery.HaveNode{Candidates: map[string]delivery.CandidateAnswer{"g": a}}) y, errB := delivery.EncodeHaveNode(delivery.HaveNode{Candidates: map[string]delivery.CandidateAnswer{"g": b}})