Merge pull request 'A node folds a placed module with no new-form build into its own composition, as the controller does, so the tunnel and the bus are not dropped when nodes compose themselves (issue 1126)' (#107) from fix/1126-fold into main

go/compose/composed.go

@@ -10,13 +10,17 @@ import (
 
 	"git.novox.be/novox/mesh-sdk/go/declaration"
 	"git.novox.be/novox/mesh-sdk/go/delivery"
+	"git.novox.be/novox/mesh-sdk/go/overlay"
 )
 
 // Compositions is one node's compositions (novox/hq ADR 0539 clauses 1 and 2): its declaration composed over in, as
 // the controller composes it, split into each module's composition at the fingerprint fps gives it and the node's own
 // (`<node>.node`), by key (delivery.ComposedKey). Each resource says its module, whatever the node reported it reads.
 // A module fps gives no fingerprint for is not composed and is named in skipped, so a caller composing one module
-// (mesh-hand-compose) gets that one. Each module's Data says whether its manifest declares data. The compositions are not
+// (mesh-hand-compose) gets that one; but for a module the node runs inside its own composition (Node.InOwn, and the
+// private network's always), which fps names at no fingerprint of the new form: its resources go into the node's own,
+// their module taken off and their ids still naming it, and its filled paths with them, as the controller's composed
+// writer folds them (issue 1126). Each module's Data says whether its manifest declares data. The compositions are not
 // signed and not yet by digest: the writer does both (delivery.SignComposed, Composed.ByDigest).
 func Compositions(in Input, node string, fps map[string]string, mode Mode) (out map[string]delivery.Composed, skipped []string, err error) {
 	out, skipped, _, err = compositions(in, node, fps, mode, false)
@@ -54,12 +58,21 @@ func compositions(in Input, node string, fps map[string]string, mode Mode, appli
 	reading := in
 	reading.Nodes = append([]Node(nil), in.Nodes...)
 	found := false
+	inOwn := map[string]bool{}
 	for i := range reading.Nodes {
 		if reading.Nodes[i].Name == node {
 			reading.Nodes[i].ReadsModules = true
 			found = true
+			for m, ok := range reading.Nodes[i].InOwn {
+				inOwn[m] = ok
+			}
 		}
 	}
+	// The private network's module is built by nobody: no fingerprint of the new form ever names it.
+	inOwn[overlay.Name] = true
+	folds := func(module string) bool {
+		return module != "" && inOwn[module] && !strings.HasPrefix(fps[module], delivery.SourcePrefix2)
+	}
 	if !found {
 		return nil, nil, nil, fmt.Errorf("compose: the mesh has no node %q", node)
 	}
@@ -101,6 +114,7 @@ func compositions(in Input, node string, fps map[string]string, mode Mode, appli
 	}
 	var kept []json.RawMessage
 	keptPaths := map[string]bool{}
+	foldedPaths := map[string]bool{}
 	left := map[string]bool{}
 	for _, raw := range envelope.Resources {
 		var head struct {
@@ -111,6 +125,23 @@ func compositions(in Input, node string, fps map[string]string, mode Mode, appli
 		if err := json.Unmarshal(raw, &head); err != nil {
 			return nil, nil, nil, err
 		}
+		if folds(head.Module) {
+			// Into the node's own, in its place among the node's resources, its module taken off (mesh-controller's
+			// compositionsOf): the node's own composition is then the controller's, byte for byte.
+			var fields map[string]json.RawMessage
+			if err := json.Unmarshal(raw, &fields); err != nil {
+				return nil, nil, nil, err
+			}
+			delete(fields, "module")
+			if raw, err = json.Marshal(fields); err != nil {
+				return nil, nil, nil, err
+			}
+			if head.Type == string(declaration.TypeFile) {
+				foldedPaths[head.Path] = true
+			}
+			kept = append(kept, raw)
+			continue
+		}
 		if _, ok := fps[head.Module]; head.Module != "" && !ok {
 			left[head.Module] = true
 			continue
@@ -120,9 +151,14 @@ func compositions(in Input, node string, fps map[string]string, mode Mode, appli
 		}
 		kept = append(kept, raw)
 	}
-	var filled []string
+	// A filled file of a module's own composition goes with that module (SplitComposed); one of a folded module goes with
+	// the node's own, after them, in the declaration's order; one of a module left out is dropped.
+	var filled, ownFilled []string
 	for _, p := range envelope.Filled {
-		if keptPaths[p] {
+		switch {
+		case foldedPaths[p]:
+			ownFilled = append(ownFilled, p)
+		case keptPaths[p]:
 			filled = append(filled, p)
 		}
 	}
@@ -131,6 +167,15 @@ func compositions(in Input, node string, fps map[string]string, mode Mode, appli
 	if err != nil {
 		return nil, nil, nil, err
 	}
+	if len(ownFilled) > 0 {
+		own, err := delivery.ComposedNodeKey(node)
+		if err != nil {
+			return nil, nil, nil, err
+		}
+		c := out[own]
+		c.Filled = append(c.Filled, ownFilled...)
+		out[own] = c
+	}
 	for key, c := range out {
 		if m, ok := in.Catalogue[c.Module]; ok && c.Module != "" {
 			c.Data = len(m.DataItems()) > 0

go/compose/composed_test.go

@@ -58,15 +58,41 @@ func TestCompositionsAreTheDeclarationSplitByModule(t *testing.T) {
 		t.Errorf("the compositions hold %v, the declaration %v", got, want)
 	}
 
-	one, skipped, err := Compositions(in, "laptop", map[string]string{"files": fpFor('b')}, Foreseeing)
-	if err != nil || strings.Join(skipped, ",") != "mesh-wireguard" {
+	one, skipped, err := Compositions(in, "laptop", map[string]string{"mesh-wireguard": fpFor('a')}, Foreseeing)
+	if err != nil || strings.Join(skipped, ",") != "files" {
 		t.Fatalf("one module: %v, skipped %v", err, skipped)
 	}
 	for _, c := range one {
-		if c.Module == "mesh-wireguard" {
+		if c.Module == "files" {
 			t.Errorf("a module given no fingerprint was composed")
 		}
 	}
+
+	// Issue 1126: a module the node runs inside its own composition (Node.InOwn), and the private network's always,
+	// given no fingerprint of the new form, is folded into the node's own as the controller's composed writer folds it:
+	// every resource in its place, its module taken off, its id still naming it, nothing skipped.
+	folding := in
+	folding.Nodes = append([]Node(nil), in.Nodes...)
+	for i := range folding.Nodes {
+		if folding.Nodes[i].Name == "laptop" {
+			folding.Nodes[i].InOwn = map[string]bool{"files": true}
+		}
+	}
+	all, skipped, err := Compositions(folding, "laptop", nil, Foreseeing)
+	nodeKey, _ := delivery.ComposedNodeKey("laptop")
+	if err != nil || len(skipped) != 0 || len(all) != 1 || len(all[nodeKey].Resources) != len(want) {
+		t.Fatalf("folded: %v, skipped %v, %d compositions, %d of %d resources in the node's own", err, skipped, len(all), len(all[nodeKey].Resources), len(want))
+	}
+	for i, raw := range all[nodeKey].Resources {
+		var r map[string]any
+		_ = json.Unmarshal(raw, &r)
+		if _, says := r["module"]; says || r["id"] != d.Resources[i]["id"] {
+			t.Errorf("folded resource %d is %v, the declaration's %v", i, r["id"], d.Resources[i]["id"])
+		}
+	}
+	if _, err := delivery.EncodeComposed(all[nodeKey]); err != nil {
+		t.Errorf("the node's own with the folded modules: %v", err)
+	}
 	if _, _, err := Compositions(in, "nowhere", fps, Foreseeing); err == nil {
 		t.Errorf("a node the mesh does not have was composed")
 	}

go/compose/frombus/frombus.go

@@ -37,6 +37,7 @@ import (
 
 	"git.novox.be/novox/mesh-sdk/go/catalogue"
 	"git.novox.be/novox/mesh-sdk/go/compose"
+	"git.novox.be/novox/mesh-sdk/go/declaration"
 	"git.novox.be/novox/mesh-sdk/go/delivery"
 	"git.novox.be/novox/mesh-sdk/go/overlay"
 )
@@ -84,11 +85,46 @@ type Mesh struct {
 // Placements, when set, stands in for the indexer's placement of the nodes it names: a node's candidate placement
 // (delivery.PlacementCandidate), so a node composes in shadow with a catalogue pull request's placement and compares
 // the result with what it runs (issue 1126); a nil value removes the node's placement. The rest is read as always.
+//
+// InOwn is the modules the composing node runs inside its own composition (InOwn of what it runs), composed there again
+// (compose.Node.InOwn) rather than cut out as modules no build is given for (issue 1126).
 type Scope struct {
 	Node       string
 	Every      bool
 	Kept       map[string]string
 	Placements map[string]*delivery.Placement
+	InOwn      []string
+}
+
+// InOwn is, of the modules p places on its node, those whose resources the node's own composition it runs
+// (`<node>.node` in running) carries, by the module their ids name (declaration.ModuleGuess), sorted: the modules the
+// controller's composed writer folded there because no fingerprint of the new form names them (mesh-wireguard, a module
+// run at a build of the old form; issue 1126). A node composing itself passes it as Scope.InOwn, so its own
+// composition keeps them as it runs them until a want names a build of the new form for one.
+func InOwn(running []delivery.Composed, p delivery.Placement) []string {
+	placed := map[string]bool{}
+	for _, m := range p.Modules {
+		placed[m.Module] = true
+	}
+	in := map[string]bool{}
+	for _, c := range running {
+		if c.Module != "" {
+			continue
+		}
+		for _, raw := range c.Resources {
+			var head struct {
+				ID     string `json:"id"`
+				Module string `json:"module"`
+			}
+			if json.Unmarshal(raw, &head) != nil || head.Module != "" {
+				continue
+			}
+			if m, ok := declaration.ModuleGuess(head.ID); ok && placed[m] {
+				in[m] = true
+			}
+		}
+	}
+	return sortedKeys(in)
 }
 
 // Input is the composition input for s, read through r.
@@ -203,6 +239,12 @@ func Input(ctx context.Context, r Reader, mesh Mesh, s Scope) (compose.Input, er
 			}
 		}
 		if node == s.Node {
+			for _, module := range s.InOwn {
+				if n.InOwn == nil {
+					n.InOwn = map[string]bool{}
+				}
+				n.InOwn[module] = true
+			}
 			for _, module := range sortedKeys(s.Kept) {
 				man, err := read(module, s.Kept[module])
 				if err != nil {

go/compose/frombus/frombus_test.go

@@ -272,6 +272,66 @@ func TestInput(t *testing.T) {
 		}
 		return out
 	}
+	// What the controller runs on n1 for a module no want names (lemurs, at a build of the old form) and the private
+	// network's (built by nobody): its composed writer (mesh-controller compositionsOf) folds their resources into the
+	// node's own, each in its place, its module taken off; the rest split by module.
+	tunnelled := func() *bus {
+		b := one("keeper", "lemurs", overlay.Name)
+		p := b.placements["n1"]
+		p.Decided.Endpoint = "n1.example:51820"
+		b.placements["n1"] = p
+		return b
+	}
+	tunnelledAs := func() map[string]*delivery.Placement {
+		p := tunnelled().placements["n1"]
+		return map[string]*delivery.Placement{"n1": &p}
+	}
+	folded := func() []delivery.Composed {
+		b := tunnelled()
+		in, err := Input(ctx, b, Mesh{}, Scope{Node: "n1"})
+		if err != nil {
+			t.Fatal(err)
+		}
+		reading := in
+		reading.Nodes = append([]compose.Node(nil), in.Nodes...)
+		reading.Nodes[0].ReadsModules = true
+		d, _, err := compose.ComposeIn(reading, "n1", compose.Reading)
+		if err != nil {
+			t.Fatal(err)
+		}
+		body, err := d.Body()
+		if err != nil {
+			t.Fatal(err)
+		}
+		var envelope struct {
+			Resources []map[string]json.RawMessage `json:"resources"`
+		}
+		if err := json.Unmarshal(body, &envelope); err != nil {
+			t.Fatal(err)
+		}
+		var resources []json.RawMessage
+		for _, r := range envelope.Resources {
+			var module string
+			_ = json.Unmarshal(r["module"], &module)
+			if module == "lemurs" || module == overlay.Name {
+				delete(r, "module")
+			}
+			raw, _ := json.Marshal(r)
+			resources = append(resources, raw)
+		}
+		split, err := delivery.SplitComposed("n1", resources, map[string]string{"keeper": oneFps["keeper"]}, nil, delivery.NodeOwn{})
+		if err != nil {
+			t.Fatal(err)
+		}
+		var out []delivery.Composed
+		for _, c := range split {
+			out = append(out, c)
+		}
+		return out
+	}()
+	if got := InOwn(folded, tunnelled().placements["n1"]); !reflect.DeepEqual(got, []string{"lemurs", overlay.Name}) {
+		t.Errorf("InOwn of the controller's composition = %v, want lemurs and %s", got, overlay.Name)
+	}
 	placedAs := func(modules ...string) map[string]*delivery.Placement {
 		p := one(modules...).placements["n1"]
 		return map[string]*delivery.Placement{"n1": &p}
@@ -315,8 +375,17 @@ func TestInput(t *testing.T) {
 			}},
 		"cannot compose": {one("keeper"), map[string]*delivery.Placement{"n1": {Node: "n2"}}, runsNow, delivery.CandidateRefused,
 			func(a delivery.CandidateAnswer) bool { return a.Refused != "" }},
+		// Issue 1126 (the seed's review): the private network's module and one at a build of the old form, which no want
+		// names and the controller folds into the node's own, are folded there again: the node's own is the same, and
+		// neither is added.
+		"mesh-wireguard and an old-form module, folded": {tunnelled(), tunnelledAs(),
+			folded, delivery.CandidateSame, func(a delivery.CandidateAnswer) bool { return len(a.Same) == 2 && len(a.Added) == 0 }},
 	} {
-		a := Answer(ctx, c.b, Mesh{}, Scope{Node: "n1", Placements: c.given}, oneFps, c.running, "h1", 7)
+		fps := oneFps
+		if name == "mesh-wireguard and an old-form module, folded" {
+			fps = map[string]string{"keeper": oneFps["keeper"]} // the node's wants: no build of lemurs or the private network
+		}
+		a := Answer(ctx, c.b, Mesh{}, Scope{Node: "n1", Placements: c.given, InOwn: InOwn(c.running, *c.given["n1"])}, fps, c.running, "h1", 7)
 		if a.Verdict != c.verdict || !c.check(a) {
 			t.Errorf("%s: %+v", name, a)
 		}

go/compose/input.go

@@ -189,6 +189,11 @@ type Node struct {
 	// Contract is the contract of its node-engine's newest statement (inv.HealthOf); Stated is whether it ever stated one.
 	Contract int
 	Stated   bool
+	// InOwn is the modules it runs inside its own composition (`<node>.node`), not by a composition of their own, since
+	// no fingerprint of the new form names them where it runs them (novox/hq issue 1126): Compositions folds their
+	// resources into the node's own, as the controller's composed writer does, unless fps gives one of the new form.
+	// The private network's module (overlay.Name), which nobody builds, is folded whether or not it is named here.
+	InOwn map[string]bool
 }
 
 // Place is where a node is, as its placement says (inv.Overlay).