novox/mesh-host
18
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

Merge pull request 'The node agent folds what it runs at no new-form build into its own composition, judged from its haves, so the first composition after a restart keeps the bus (issue 1126)' (#152) from fix/1126-fold into main

Browse Source
This commit was merged in pull request #152.
This commit is contained in:
mesh-tools
2026-10-27 08:09:06 +00:00
parent 0d22776b20 69b845c0f7
commit 50c229f905
5 changed files with 150 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
go.mod
+1 -1
View File
@@ -3,7 +3,7 @@ module github.com/novox/mesh-host
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027014310-cadd7b1b2f8f
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027075026-05b678310366
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
golang.org/x/crypto v0.57.0
go.sum
+2
View File
@@ -10,6 +10,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027012704-062c29c3a22f h1:AKzV973U
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027012704-062c29c3a22f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027014310-cadd7b1b2f8f h1:mimAGERguQmCGYVNdRTrFcZ1dhpU6GWo+Q3EDhHOrVk=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027014310-cadd7b1b2f8f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027075026-05b678310366 h1:B7oQ82ar2McqZ9OrbJfyvG+50Ol6fBUoK3eYE2uTPvo=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027075026-05b678310366/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
internal/selfcompose/selfcompose.go
+72 -11
View File
@@ -71,10 +71,13 @@ type Composer struct {
wants map[string]delivery.Want // by module, the node's own under agent.NodeModule
placements map[string]delivery.Placement
records map[string]delivery.NodeRecord
modules map[string]delivery.Module
values map[string]map[string]any // by module, this node's
manifests map[string][]byte // by digest, read once
handed map[string]string // composed key to the digest handed
// inOwn is, by module, whether this node's have of it names no fingerprint of the new form (mesh-wireguard, a module
// at a build of the old form): the node runs it inside its own composition, and composes it there again (issue 1126).
inOwn map[string]bool
modules map[string]delivery.Module
values map[string]map[string]any // by module, this node's
manifests map[string][]byte // by digest, read once
handed map[string]string // composed key to the digest handed
// sealedOwn is each own secret's sealed value as first handed, by module/name and generation: a value sealed again
// is other bytes, so the one first had is kept while the generation stands, and the composition stays the same.
sealedOwn map[string]ownSealed
@@ -90,7 +93,7 @@ func New(env Env) *Composer {
env.Say = func(string) {}
}
return &Composer{env: env, wants: map[string]delivery.Want{}, placements: map[string]delivery.Placement{},
records: map[string]delivery.NodeRecord{}, modules: map[string]delivery.Module{}, values: map[string]map[string]any{},
records: map[string]delivery.NodeRecord{}, inOwn: map[string]bool{}, modules: map[string]delivery.Module{}, values: map[string]map[string]any{},
manifests: map[string][]byte{}, handed: map[string]string{}, sealedOwn: map[string]ownSealed{}}
}
@@ -105,16 +108,19 @@ type ownSealed struct {
type input struct {
bucket, pattern string
take func(c *Composer, u agent.Update)
// changed, when set, takes the update in take's place and says whether it changed anything composed.
changed func(c *Composer, u agent.Update) bool
}
func (c *Composer) inputs() []input {
node := c.env.Node
return []input{
{delivery.AssignerWantsBucket, "want." + node + ".>", (*Composer).takeWant},
{delivery.PlacementBucket, ">", (*Composer).takePlacement},
{delivery.HaveBucket, "*." + delivery.HaveNodeWord, (*Composer).takeRecord},
{delivery.ModulesBucket, ">", (*Composer).takeModule},
{ResolvedBucket, node + ".>", (*Composer).takeValue},
{delivery.AssignerWantsBucket, "want." + node + ".>", (*Composer).takeWant, nil},
{delivery.PlacementBucket, ">", (*Composer).takePlacement, nil},
// Every node's record, and this node's haves: what it runs inside its own composition.
{delivery.HaveBucket, ">", nil, (*Composer).takeHave},
{delivery.ModulesBucket, ">", (*Composer).takeModule, nil},
{ResolvedBucket, node + ".>", (*Composer).takeValue, nil},
}
}
@@ -193,6 +199,8 @@ func (c *Composer) Watch(ctx context.Context, _ string) (<-chan agent.Update, er
dirty = true // a secret changed: its new generation is asked of the vault at the compose
case t.u.Caught:
caught[t.in] = true
case ins[t.in].changed != nil:
dirty = ins[t.in].changed(c, t.u) || dirty
default:
ins[t.in].take(c, t.u)
dirty = true
@@ -302,6 +310,56 @@ func (c *Composer) takeRecord(u agent.Update) {
c.records[node] = *h.Record
}
// takeHave takes a have key: a node's record (always composed again), or one of this node's modules' haves, which
// changes what is composed only when whether it runs inside the node's own composition changes.
func (c *Composer) takeHave(u agent.Update) bool {
if isNodeKey(u.Key) {
c.takeRecord(u)
return true
}
return c.takeModuleHave(u)
}
// takeModuleHave takes one of this node's modules' haves, and says whether it changed which it runs inside its own
// composition: one whose have names no fingerprint of the new form and is not removed (nats on novox: a build-...
// id; mesh-wireguard: none). Read from the haves, which outlast a restart, never from what the agent has applied
// this run: a fresh agent composes before it applies anything (review of the fold, round 1).
func (c *Composer) takeModuleHave(u agent.Update) bool {
node, module, isNode, err := delivery.ParseHaveKey(u.Key)
if err != nil || isNode || node != c.env.Node {
return false
}
in := false
if !u.Deleted {
if h, err := delivery.DecodeHave(u.Value); err == nil {
in = h.Phase != delivery.PhaseRemoved && !strings.HasPrefix(h.FP, delivery.SourcePrefix2)
}
}
c.mu.Lock()
defer c.mu.Unlock()
if c.inOwn[module] == in {
return false
}
if in {
c.inOwn[module] = true
} else {
delete(c.inOwn, module)
}
return true
}
// inOwnOf is, of the modules p places, those the node runs inside its own composition (c.inOwn), sorted; c.mu held.
func (c *Composer) inOwnOf(p delivery.Placement) []string {
var out []string
for _, m := range p.Modules {
if c.inOwn[m.Module] {
out = append(out, m.Module)
}
}
sort.Strings(out)
return out
}
func (c *Composer) takeModule(u agent.Update) {
name, err := delivery.ParseModulesKey(u.Key)
if err != nil {
@@ -359,6 +417,9 @@ func (c *Composer) compose(ctx context.Context) []agent.Update {
}
}
_, nodeWanted := c.wants[agent.NodeModule]
// What it runs inside its own composition and no want names at a build of the new form stays there, as the
// controller's composed writer folds it (issue 1126); compose folds none a want names.
inOwn := c.inOwnOf(c.placements[c.env.Node])
r := reader{c: c}
c.mu.Unlock()
sort.Strings(behind)
@@ -373,7 +434,7 @@ func (c *Composer) compose(ctx context.Context) []agent.Update {
kept[module] = digest
}
compositions, _, err := frombus.Compose(ctx, r, c.env.Mesh, frombus.Scope{Node: c.env.Node, Kept: kept}, fps)
compositions, _, err := frombus.Compose(ctx, r, c.env.Mesh, frombus.Scope{Node: c.env.Node, Kept: kept, InOwn: inOwn}, fps)
if err != nil {
c.refused(fmt.Sprintf("%v", err))
c.sayOnce(fmt.Sprintf("the node cannot compose its modules, and keeps running what it runs: %v", err))
internal/selfcompose/selfcompose_test.go
+35 -2
View File
@@ -112,21 +112,33 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
if err != nil {
t.Fatal(err)
}
// oldie is placed and no want names it (a module at a build of the old form, nats on novox): the node runs it inside
// its own composition, and composes it there again, even before the agent has applied anything (Running empty).
oldie := `{"module":"oldie","version":"1","capabilities":["systemd"],"resources":[{"id":"service","type":"service","unit":"oldie.service","state":"running","boot":"enabled"}]}`
odigest := delivery.ManifestDigest([]byte(oldie))
oman, _ := delivery.EncodeManifest(delivery.Manifest{Module: "oldie", JSON: oldie})
okey, _ := delivery.ManifestsKey(odigest)
b.kv[delivery.ManifestsBucket+"/"+okey] = oman
omod, _ := delivery.EncodeModule(delivery.Module{FP2: delivery.SourcePrefix2 + strings.Repeat("c", 64), Manifest: odigest, Repository: "novox/o", Commit: "c"})
place, _ := delivery.EncodePlacement(delivery.Placement{Node: "n1", Decided: delivery.NodeDecided{Hub: true, Address: "10.77.0.1"},
Modules: []delivery.PlacedModule{{Module: "lemurs"}}})
Modules: []delivery.PlacedModule{{Module: "lemurs"}, {Module: "oldie"}}})
have, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: "n1", TunnelKey: "k", SealingKey: "sk", Capabilities: []string{"systemd"}}})
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: digest, Repository: "novox/x", Commit: "c"})
want, _ := delivery.EncodeWant(delivery.Want{FP: fp, Self: true})
nodeWant, _ := delivery.EncodeNodeWant(delivery.NodeWant{Self: true})
b.put(t, delivery.PlacementBucket, "placement.n1", place)
b.put(t, delivery.HaveBucket, "n1.node", have)
oldHave, _ := delivery.EncodeHave(delivery.Have{Build: "build-1", Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
b.put(t, delivery.HaveBucket, "n1.oldie", oldHave)
b.put(t, delivery.ModulesBucket, "lemurs", mod)
b.put(t, delivery.ModulesBucket, "oldie", omod)
b.put(t, delivery.AssignerWantsBucket, "want.n1.lemurs", want)
b.put(t, delivery.AssignerWantsBucket, "want.n1.node", nodeWant)
for _, bucket := range []string{delivery.AssignerWantsBucket, delivery.PlacementBucket, delivery.HaveBucket, delivery.ModulesBucket, ResolvedBucket} {
b.ch(bucket) <- agent.Update{Caught: true}
}
got := map[string]bool{}
var own delivery.Composed
for {
u := next(t, out)
if u.Caught {
@@ -139,8 +151,18 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
if c.Node != "n1" {
t.Errorf("%s composed for %q", u.Key, c.Node)
}
if c.Module == "" {
own = c
}
got[u.Key] = true
}
var ownText []string
for _, r := range own.Resources {
ownText = append(ownText, string(r))
}
if !strings.Contains(strings.Join(ownText, " "), `"id":"oldie.service"`) {
t.Errorf("the node's own composition, composed before anything was applied, lacks oldie: %v", ownText)
}
lemursKey, _ := delivery.ComposedKey("n1", "lemurs", fp)
nodeKey, _ := delivery.ComposedNodeKey("n1")
if !got[lemursKey] || !got[nodeKey] || len(got) != 2 {
@@ -196,7 +218,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
b.mu.Unlock()
kmod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: kdigest, Repository: "novox/k", Commit: "c"})
both, _ := delivery.EncodePlacement(delivery.Placement{Node: "n1", Decided: delivery.NodeDecided{Hub: true, Address: "10.77.0.1"},
Modules: []delivery.PlacedModule{{Module: "lemurs"}, {Module: "keeper"}}})
Modules: []delivery.PlacedModule{{Module: "lemurs"}, {Module: "oldie"}, {Module: "keeper"}}})
b.put(t, delivery.ModulesBucket, "keeper", kmod)
b.put(t, delivery.PlacementBucket, "placement.n1", both)
b.put(t, delivery.AssignerWantsBucket, "want.n1.keeper", want)
@@ -444,6 +466,17 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
_, ok := a["drops"]
return !ok && len(a) == 2
})
// Issue 1126, the seed's review: a module n1 runs inside its own composition (nats, at a build of the old form) is
// composed there again, not at the trunk's build as one added; lemurs at the build it runs, keeper (added) at the
// trunk's.
inOwnRunning := []delivery.Composed{{Node: "n1", Module: "lemurs", FP: fp}}
other := "src2:" + strings.Repeat("9", 64)
gotFps := shadowBuilds(inOwnRunning, delivery.Placement{Node: "n1", Modules: []delivery.PlacedModule{{Module: "lemurs"}, {Module: "nats"}, {Module: "keeper"}}},
map[string]delivery.Module{"lemurs": {FP2: other}, "nats": {FP2: other}, "keeper": {FP2: other}}, []string{"nats"})
if fmt.Sprint(gotFps) != fmt.Sprint(map[string]string{"lemurs": fp, "keeper": other}) {
t.Errorf("shadow builds %v; want lemurs at what runs, keeper at the trunk's, nats at none (folded)", gotFps)
}
}
// patternBus is a bus whose every watch gets the updates of its bucket matching its pattern, as the bus's own watches
internal/selfcompose/shadow.go
+40 -22
View File
@@ -89,13 +89,13 @@ func (s *Shadow) Run(ctx context.Context) error {
c := s.c
node := c.env.Node
ins := []input{
{delivery.PlacementCandidatesBucket, ">", nil},
{delivery.PlacementBucket, ">", (*Composer).takePlacement},
{delivery.HaveBucket, "*." + delivery.HaveNodeWord, nil},
{delivery.ModulesBucket, ">", (*Composer).takeModule},
{ResolvedBucket, node + ".>", (*Composer).takeValue},
{delivery.PlacementCandidatesBucket, ">", nil, nil},
{delivery.PlacementBucket, ">", (*Composer).takePlacement, nil},
{delivery.HaveBucket, "*." + delivery.HaveNodeWord, nil, nil},
{delivery.ModulesBucket, ">", (*Composer).takeModule, nil},
{ResolvedBucket, node + ".>", (*Composer).takeValue, nil},
// What this node runs moved: an apply ends in its module's have.
{delivery.HaveBucket, node + ".*", nil},
{delivery.HaveBucket, node + ".*", nil, nil},
}
type tagged struct {
in int
@@ -140,6 +140,7 @@ func (s *Shadow) Run(ctx context.Context) error {
case t.in == 2:
dirty = s.takeRecord(t.u) || dirty
case t.in == 5:
c.takeModuleHave(t.u)
dirty = !isNodeKey(t.u.Key) || dirty
default:
ins[t.in].take(c, t.u)
@@ -305,14 +306,6 @@ func (s *Shadow) answerGroup(ctx context.Context, group string) (delivery.Candid
return a, true
}
// Each module at the build it runs; one the candidate adds at the trunk's build.
fps := map[string]string{}
kept := map[string]string{}
for _, r := range running {
if r.Module != "" && r.FP != "" {
fps[r.Module] = r.FP
}
}
placement := given[node]
if placement == nil {
c.mu.Lock()
@@ -320,13 +313,12 @@ func (s *Shadow) answerGroup(ctx context.Context, group string) (delivery.Candid
c.mu.Unlock()
placement = &p
}
for _, m := range placement.Modules {
if _, runs := fps[m.Module]; !runs {
if mod, ok := modules[m.Module]; ok && mod.FP2 != "" {
fps[m.Module] = mod.FP2
}
}
}
// What it runs inside its own composition, of what the candidate places (the seed's trunk places nothing).
c.mu.Lock()
inOwn := c.inOwnOf(*placement)
c.mu.Unlock()
fps := shadowBuilds(running, *placement, modules, inOwn)
kept := map[string]string{}
for module, fp := range fps {
if mod, ok := modules[module]; ok && mod.FP2 == fp {
continue
@@ -347,10 +339,36 @@ func (s *Shadow) answerGroup(ctx context.Context, group string) (delivery.Candid
c.mu.Lock()
c.wants = held
c.mu.Unlock()
a := frombus.Answer(ctx, reader{c: c}, c.env.Mesh, frombus.Scope{Node: node, Kept: kept, Placements: given}, fps, running, head, rev)
a := frombus.Answer(ctx, reader{c: c}, c.env.Mesh, frombus.Scope{Node: node, Kept: kept, Placements: given, InOwn: inOwn}, fps, running, head, rev)
return a, true
}
// shadowBuilds is the build each module placed is composed at in shadow: the one it runs, and for one the candidate
// adds, the trunk's. A module it runs inside its own composition (inOwn, from its haves: mesh-wireguard, one at a build
// of the old form) is given none, so it is folded there again, as the node's own composer folds it, rather than
// composed at the trunk's build as if the candidate added it (issue 1126: nats and photos have a trunk build that no
// want names).
func shadowBuilds(running []delivery.Composed, placement delivery.Placement, modules map[string]delivery.Module, inOwn []string) map[string]string {
fps := map[string]string{}
for _, r := range running {
if r.Module != "" && r.FP != "" {
fps[r.Module] = r.FP
}
}
folded := map[string]bool{}
for _, m := range inOwn {
folded[m] = true
}
for _, m := range placement.Modules {
if _, runs := fps[m.Module]; !runs && !folded[m.Module] {
if mod, ok := modules[m.Module]; ok && mod.FP2 != "" {
fps[m.Module] = mod.FP2
}
}
}
return fps
}
func sameAnswer(a, b delivery.CandidateAnswer) bool {
x, errA := delivery.EncodeHaveNode(delivery.HaveNode{Candidates: map[string]delivery.CandidateAnswer{"g": a}})
y, errB := delivery.EncodeHaveNode(delivery.HaveNode{Candidates: map[string]delivery.CandidateAnswer{"g": b}})