diff --git a/internal/selfcompose/selfcompose.go b/internal/selfcompose/selfcompose.go index 3561df3..24f190f 100644 --- a/internal/selfcompose/selfcompose.go +++ b/internal/selfcompose/selfcompose.go @@ -71,10 +71,13 @@ type Composer struct { wants map[string]delivery.Want // by module, the node's own under agent.NodeModule placements map[string]delivery.Placement records map[string]delivery.NodeRecord - modules map[string]delivery.Module - values map[string]map[string]any // by module, this node's - manifests map[string][]byte // by digest, read once - handed map[string]string // composed key to the digest handed + // inOwn is, by module, whether this node's have of it names no fingerprint of the new form (mesh-wireguard, a module + // at a build of the old form): the node runs it inside its own composition, and composes it there again (issue 1126). + inOwn map[string]bool + modules map[string]delivery.Module + values map[string]map[string]any // by module, this node's + manifests map[string][]byte // by digest, read once + handed map[string]string // composed key to the digest handed // sealedOwn is each own secret's sealed value as first handed, by module/name and generation: a value sealed again // is other bytes, so the one first had is kept while the generation stands, and the composition stays the same. sealedOwn map[string]ownSealed @@ -90,7 +93,7 @@ func New(env Env) *Composer { env.Say = func(string) {} } return &Composer{env: env, wants: map[string]delivery.Want{}, placements: map[string]delivery.Placement{}, - records: map[string]delivery.NodeRecord{}, modules: map[string]delivery.Module{}, values: map[string]map[string]any{}, + records: map[string]delivery.NodeRecord{}, inOwn: map[string]bool{}, modules: map[string]delivery.Module{}, values: map[string]map[string]any{}, manifests: map[string][]byte{}, handed: map[string]string{}, sealedOwn: map[string]ownSealed{}} } @@ -105,16 +108,19 @@ type ownSealed struct { type input struct { bucket, pattern string take func(c *Composer, u agent.Update) + // changed, when set, takes the update in take's place and says whether it changed anything composed. + changed func(c *Composer, u agent.Update) bool } func (c *Composer) inputs() []input { node := c.env.Node return []input{ - {delivery.AssignerWantsBucket, "want." + node + ".>", (*Composer).takeWant}, - {delivery.PlacementBucket, ">", (*Composer).takePlacement}, - {delivery.HaveBucket, "*." + delivery.HaveNodeWord, (*Composer).takeRecord}, - {delivery.ModulesBucket, ">", (*Composer).takeModule}, - {ResolvedBucket, node + ".>", (*Composer).takeValue}, + {delivery.AssignerWantsBucket, "want." + node + ".>", (*Composer).takeWant, nil}, + {delivery.PlacementBucket, ">", (*Composer).takePlacement, nil}, + // Every node's record, and this node's haves: what it runs inside its own composition. + {delivery.HaveBucket, ">", nil, (*Composer).takeHave}, + {delivery.ModulesBucket, ">", (*Composer).takeModule, nil}, + {ResolvedBucket, node + ".>", (*Composer).takeValue, nil}, } } @@ -193,6 +199,8 @@ func (c *Composer) Watch(ctx context.Context, _ string) (<-chan agent.Update, er dirty = true // a secret changed: its new generation is asked of the vault at the compose case t.u.Caught: caught[t.in] = true + case ins[t.in].changed != nil: + dirty = ins[t.in].changed(c, t.u) || dirty default: ins[t.in].take(c, t.u) dirty = true @@ -302,6 +310,56 @@ func (c *Composer) takeRecord(u agent.Update) { c.records[node] = *h.Record } +// takeHave takes a have key: a node's record (always composed again), or one of this node's modules' haves, which +// changes what is composed only when whether it runs inside the node's own composition changes. +func (c *Composer) takeHave(u agent.Update) bool { + if isNodeKey(u.Key) { + c.takeRecord(u) + return true + } + return c.takeModuleHave(u) +} + +// takeModuleHave takes one of this node's modules' haves, and says whether it changed which it runs inside its own +// composition: one whose have names no fingerprint of the new form and is not removed (nats on novox: a build-... +// id; mesh-wireguard: none). Read from the haves, which outlast a restart, never from what the agent has applied +// this run: a fresh agent composes before it applies anything (review of the fold, round 1). +func (c *Composer) takeModuleHave(u agent.Update) bool { + node, module, isNode, err := delivery.ParseHaveKey(u.Key) + if err != nil || isNode || node != c.env.Node { + return false + } + in := false + if !u.Deleted { + if h, err := delivery.DecodeHave(u.Value); err == nil { + in = h.Phase != delivery.PhaseRemoved && !strings.HasPrefix(h.FP, delivery.SourcePrefix2) + } + } + c.mu.Lock() + defer c.mu.Unlock() + if c.inOwn[module] == in { + return false + } + if in { + c.inOwn[module] = true + } else { + delete(c.inOwn, module) + } + return true +} + +// inOwnOf is, of the modules p places, those the node runs inside its own composition (c.inOwn), sorted; c.mu held. +func (c *Composer) inOwnOf(p delivery.Placement) []string { + var out []string + for _, m := range p.Modules { + if c.inOwn[m.Module] { + out = append(out, m.Module) + } + } + sort.Strings(out) + return out +} + func (c *Composer) takeModule(u agent.Update) { name, err := delivery.ParseModulesKey(u.Key) if err != nil { @@ -359,15 +417,11 @@ func (c *Composer) compose(ctx context.Context) []agent.Update { } } _, nodeWanted := c.wants[agent.NodeModule] - placement := c.placements[c.env.Node] + // What it runs inside its own composition and no want names at a build of the new form stays there, as the + // controller's composed writer folds it (issue 1126); compose folds none a want names. + inOwn := c.inOwnOf(c.placements[c.env.Node]) r := reader{c: c} c.mu.Unlock() - // What it runs inside its own composition and no want names (mesh-wireguard, a module at a build of the old form) - // stays there, as the controller's composed writer folds it (issue 1126). - var inOwn []string - if c.env.Running != nil { - inOwn = frombus.InOwn(c.env.Running(), placement) - } sort.Strings(behind) for _, module := range behind { digest, err := c.builtManifest(ctx, module, fps[module]) diff --git a/internal/selfcompose/selfcompose_test.go b/internal/selfcompose/selfcompose_test.go index 321ab00..3175f53 100644 --- a/internal/selfcompose/selfcompose_test.go +++ b/internal/selfcompose/selfcompose_test.go @@ -112,21 +112,33 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { if err != nil { t.Fatal(err) } + // oldie is placed and no want names it (a module at a build of the old form, nats on novox): the node runs it inside + // its own composition, and composes it there again, even before the agent has applied anything (Running empty). + oldie := `{"module":"oldie","version":"1","capabilities":["systemd"],"resources":[{"id":"service","type":"service","unit":"oldie.service","state":"running","boot":"enabled"}]}` + odigest := delivery.ManifestDigest([]byte(oldie)) + oman, _ := delivery.EncodeManifest(delivery.Manifest{Module: "oldie", JSON: oldie}) + okey, _ := delivery.ManifestsKey(odigest) + b.kv[delivery.ManifestsBucket+"/"+okey] = oman + omod, _ := delivery.EncodeModule(delivery.Module{FP2: delivery.SourcePrefix2 + strings.Repeat("c", 64), Manifest: odigest, Repository: "novox/o", Commit: "c"}) place, _ := delivery.EncodePlacement(delivery.Placement{Node: "n1", Decided: delivery.NodeDecided{Hub: true, Address: "10.77.0.1"}, - Modules: []delivery.PlacedModule{{Module: "lemurs"}}}) + Modules: []delivery.PlacedModule{{Module: "lemurs"}, {Module: "oldie"}}}) have, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: "n1", TunnelKey: "k", SealingKey: "sk", Capabilities: []string{"systemd"}}}) mod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: digest, Repository: "novox/x", Commit: "c"}) want, _ := delivery.EncodeWant(delivery.Want{FP: fp, Self: true}) nodeWant, _ := delivery.EncodeNodeWant(delivery.NodeWant{Self: true}) b.put(t, delivery.PlacementBucket, "placement.n1", place) b.put(t, delivery.HaveBucket, "n1.node", have) + oldHave, _ := delivery.EncodeHave(delivery.Have{Build: "build-1", Phase: delivery.PhaseRunning, Health: delivery.HealthOK}) + b.put(t, delivery.HaveBucket, "n1.oldie", oldHave) b.put(t, delivery.ModulesBucket, "lemurs", mod) + b.put(t, delivery.ModulesBucket, "oldie", omod) b.put(t, delivery.AssignerWantsBucket, "want.n1.lemurs", want) b.put(t, delivery.AssignerWantsBucket, "want.n1.node", nodeWant) for _, bucket := range []string{delivery.AssignerWantsBucket, delivery.PlacementBucket, delivery.HaveBucket, delivery.ModulesBucket, ResolvedBucket} { b.ch(bucket) <- agent.Update{Caught: true} } got := map[string]bool{} + var own delivery.Composed for { u := next(t, out) if u.Caught { @@ -139,8 +151,18 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { if c.Node != "n1" { t.Errorf("%s composed for %q", u.Key, c.Node) } + if c.Module == "" { + own = c + } got[u.Key] = true } + var ownText []string + for _, r := range own.Resources { + ownText = append(ownText, string(r)) + } + if !strings.Contains(strings.Join(ownText, " "), `"id":"oldie.service"`) { + t.Errorf("the node's own composition, composed before anything was applied, lacks oldie: %v", ownText) + } lemursKey, _ := delivery.ComposedKey("n1", "lemurs", fp) nodeKey, _ := delivery.ComposedNodeKey("n1") if !got[lemursKey] || !got[nodeKey] || len(got) != 2 { @@ -196,7 +218,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { b.mu.Unlock() kmod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: kdigest, Repository: "novox/k", Commit: "c"}) both, _ := delivery.EncodePlacement(delivery.Placement{Node: "n1", Decided: delivery.NodeDecided{Hub: true, Address: "10.77.0.1"}, - Modules: []delivery.PlacedModule{{Module: "lemurs"}, {Module: "keeper"}}}) + Modules: []delivery.PlacedModule{{Module: "lemurs"}, {Module: "oldie"}, {Module: "keeper"}}}) b.put(t, delivery.ModulesBucket, "keeper", kmod) b.put(t, delivery.PlacementBucket, "placement.n1", both) b.put(t, delivery.AssignerWantsBucket, "want.n1.keeper", want) @@ -448,13 +470,12 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { // Issue 1126, the seed's review: a module n1 runs inside its own composition (nats, at a build of the old form) is // composed there again, not at the trunk's build as one added; lemurs at the build it runs, keeper (added) at the // trunk's. - inOwnRunning := []delivery.Composed{{Node: "n1", Resources: []json.RawMessage{json.RawMessage(`{"id":"nats.server","type":"service"}`)}}, - {Node: "n1", Module: "lemurs", FP: fp}} + inOwnRunning := []delivery.Composed{{Node: "n1", Module: "lemurs", FP: fp}} other := "src2:" + strings.Repeat("9", 64) - gotFps, gotInOwn := shadowBuilds(inOwnRunning, delivery.Placement{Node: "n1", Modules: []delivery.PlacedModule{{Module: "lemurs"}, {Module: "nats"}, {Module: "keeper"}}}, - map[string]delivery.Module{"lemurs": {FP2: other}, "nats": {FP2: other}, "keeper": {FP2: other}}) - if fmt.Sprint(gotFps) != fmt.Sprint(map[string]string{"lemurs": fp, "keeper": other}) || fmt.Sprint(gotInOwn) != "[nats]" { - t.Errorf("shadow builds %v, in its own %v; want lemurs at what runs, keeper at the trunk's, nats in its own", gotFps, gotInOwn) + gotFps := shadowBuilds(inOwnRunning, delivery.Placement{Node: "n1", Modules: []delivery.PlacedModule{{Module: "lemurs"}, {Module: "nats"}, {Module: "keeper"}}}, + map[string]delivery.Module{"lemurs": {FP2: other}, "nats": {FP2: other}, "keeper": {FP2: other}}, []string{"nats"}) + if fmt.Sprint(gotFps) != fmt.Sprint(map[string]string{"lemurs": fp, "keeper": other}) { + t.Errorf("shadow builds %v; want lemurs at what runs, keeper at the trunk's, nats at none (folded)", gotFps) } } diff --git a/internal/selfcompose/shadow.go b/internal/selfcompose/shadow.go index 23f72d8..e03cc51 100644 --- a/internal/selfcompose/shadow.go +++ b/internal/selfcompose/shadow.go @@ -89,13 +89,13 @@ func (s *Shadow) Run(ctx context.Context) error { c := s.c node := c.env.Node ins := []input{ - {delivery.PlacementCandidatesBucket, ">", nil}, - {delivery.PlacementBucket, ">", (*Composer).takePlacement}, - {delivery.HaveBucket, "*." + delivery.HaveNodeWord, nil}, - {delivery.ModulesBucket, ">", (*Composer).takeModule}, - {ResolvedBucket, node + ".>", (*Composer).takeValue}, + {delivery.PlacementCandidatesBucket, ">", nil, nil}, + {delivery.PlacementBucket, ">", (*Composer).takePlacement, nil}, + {delivery.HaveBucket, "*." + delivery.HaveNodeWord, nil, nil}, + {delivery.ModulesBucket, ">", (*Composer).takeModule, nil}, + {ResolvedBucket, node + ".>", (*Composer).takeValue, nil}, // What this node runs moved: an apply ends in its module's have. - {delivery.HaveBucket, node + ".*", nil}, + {delivery.HaveBucket, node + ".*", nil, nil}, } type tagged struct { in int @@ -140,6 +140,7 @@ func (s *Shadow) Run(ctx context.Context) error { case t.in == 2: dirty = s.takeRecord(t.u) || dirty case t.in == 5: + c.takeModuleHave(t.u) dirty = !isNodeKey(t.u.Key) || dirty default: ins[t.in].take(c, t.u) @@ -312,7 +313,11 @@ func (s *Shadow) answerGroup(ctx context.Context, group string) (delivery.Candid c.mu.Unlock() placement = &p } - fps, inOwn := shadowBuilds(running, *placement, modules) + // What it runs inside its own composition, of what the candidate places (the seed's trunk places nothing). + c.mu.Lock() + inOwn := c.inOwnOf(*placement) + c.mu.Unlock() + fps := shadowBuilds(running, *placement, modules, inOwn) kept := map[string]string{} for module, fp := range fps { if mod, ok := modules[module]; ok && mod.FP2 == fp { @@ -339,17 +344,17 @@ func (s *Shadow) answerGroup(ctx context.Context, group string) (delivery.Candid } // shadowBuilds is the build each module placed is composed at in shadow: the one it runs, and for one the candidate -// adds, the trunk's; and inOwn, the modules it runs inside its own composition (mesh-wireguard, one at a build of the -// old form), which are composed there again rather than at the trunk's build as if the candidate added them (issue -// 1126: nats and photos have a trunk build that no want names, and the seed's answer named them added). -func shadowBuilds(running []delivery.Composed, placement delivery.Placement, modules map[string]delivery.Module) (map[string]string, []string) { +// adds, the trunk's. A module it runs inside its own composition (inOwn, from its haves: mesh-wireguard, one at a build +// of the old form) is given none, so it is folded there again, as the node's own composer folds it, rather than +// composed at the trunk's build as if the candidate added it (issue 1126: nats and photos have a trunk build that no +// want names). +func shadowBuilds(running []delivery.Composed, placement delivery.Placement, modules map[string]delivery.Module, inOwn []string) map[string]string { fps := map[string]string{} for _, r := range running { if r.Module != "" && r.FP != "" { fps[r.Module] = r.FP } } - inOwn := frombus.InOwn(running, placement) folded := map[string]bool{} for _, m := range inOwn { folded[m] = true @@ -361,7 +366,7 @@ func shadowBuilds(running []delivery.Composed, placement delivery.Placement, mod } } } - return fps, inOwn + return fps } func sameAnswer(a, b delivery.CandidateAnswer) bool {