Merge pull request 'frombus: a node composing itself takes its modules from their builds and reads what providers elsewhere serve (issue 1126, group fix/1126-node-creds)' (#109) from fix/1126-node-creds into main

go/buildstep/launchers.go

@@ -1,24 +1,14 @@
 package buildstep
 
-import "strings"
+import "git.novox.be/novox/mesh-sdk/go/catalogue"
 
 // LauncherSuffix is what a TypeScript entrypoint's launcher is called beside it: index.js is started as
 // index.serve.mjs (novox/hq ADR 0193), an ES module by its own extension whatever the bundle's package.json says.
-const LauncherSuffix = ".serve.mjs"
+const LauncherSuffix = catalogue.LauncherSuffix
 
-// Launchers are the launchers a bundle's build writes, by entrypoint, relative to the bundle's root: for TypeScript
+// Launchers are the launchers a bundle's build writes, by entrypoint, relative to the bundle's root
-// every entrypoint `x.js` is started by `x.serve.mjs` beside it; an entrypoint that is not JavaScript and every other
+// (catalogue.Launchers): the one rule mesh-build-node, build-agent, the controller and a node taking in a build share
-// language, whose build is already executable, get none. The one rule mesh-build-node, build-agent and the controller
+// (R4e, issue 1126), so the name a build writes is the name a reader expects.
-// share (R4e), so the name a build writes is the name a reader expects.
 func Launchers(language string, entrypoints []string) map[string]string {
-	out := map[string]string{}
+	return catalogue.Launchers(language, entrypoints)
-	if language != "typescript" {
-		return out
-	}
-	for _, entry := range entrypoints {
-		if strings.HasSuffix(entry, ".js") {
-			out[entry] = strings.TrimSuffix(entry, ".js") + LauncherSuffix
-		}
-	}
-	return out
 }

go/buscompose/buscompose_test.go

@@ -223,6 +223,31 @@ func TestTheCopiedComposerComposesAsTheControllers(t *testing.T) {
 		if !slices.Contains(perms.Subscribe, VaultSecretChanged) {
 			t.Errorf("node one does not hear the vault's secret-changed: %v", perms.Subscribe)
 		}
+		// And what a module on another node serves from its settings, that key alone (mailu's smtp domain, issue 1126).
+		users, err := Users(Records{Nodes: []string{"one", "two"}, Assigned: map[string][]Declared{
+			"two": {{Module: "mailer", Served: []string{"domain", "not.a.token"}}}}})
+		if err != nil {
+			t.Fatal(err)
+		}
+		get := func(key string) string {
+			return "$JS.API.DIRECT.GET.KV_" + ResolvedBucket + ".$KV." + ResolvedBucket + "." + key
+		}
+		for _, u := range users {
+			if u.Kind != KindNode {
+				continue
+			}
+			perms, err := PermissionsFor(u)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if u.Node == "one" && (!reflect.DeepEqual(u.ServedReads, []string{"two.mailer.domain"}) || !MayPublish(perms, get("two.mailer.domain")) ||
+				MayPublish(perms, get("two.mailer.sitename"))) {
+				t.Errorf("node one reads %v of two's resolved values", u.ServedReads)
+			}
+			if u.Node == "two" && len(u.ServedReads) != 0 {
+				t.Errorf("node two is granted its own served keys apart: %v", u.ServedReads)
+			}
+		}
 	})
 	t.Run("the controller's golden server configuration", func(t *testing.T) {
 		seat := Seat{Name: "telegram-sender", Accepts: []string{"send"}, Emits: []string{"delivered", "failed"}}

go/buscompose/nats.go

@@ -217,6 +217,10 @@ type Principal struct {
 	// ADR 0547 §2, ADR 0550 §5, issue 1126): its modules' own secrets sealed to its key, for the compositions it makes
 	// itself. Set while the vault's holder serves it.
 	AsksItsOwnSecrets bool
+	// ServedReads are, for a machine principal, the keys of mesh-config's resolved values that other nodes' modules serve
+	// from (`<node>.<module>.<key>`, catalogue.ServedSettings): what every consumer is told, and so what a node composing
+	// its own modules reads of a provider elsewhere (issue 1126), each key alone. Sorted, none twice.
+	ServedReads []string
 	// FollowsGrantsUserList is the machine principal of the node holding mesh-broker: its node-engine reads the grants
 	// part's sealed user list, one key and an object by name (novox/hq ADR 0547 clause 7, GrantsUserListFollows).
 	FollowsGrantsUserList bool
@@ -898,6 +902,12 @@ func PermissionsFor(p Principal) (Permissions, error) {
 		// manifests, the builder's builds (the manifest of a build its want names), and mesh-config's resolved values of
 		// its own node alone. Read only.
 		pub = append(pub, NodeComposeReads(p.Node)...)
+		// And, of the other nodes' resolved values, the keys their modules serve from (issue 1126), each one key.
+		served, err := ServedReadGrants(p.ServedReads)
+		if err != nil {
+			return Permissions{}, err
+		}
+		pub = append(pub, served...)
 		// And an object of the declaration bucket by its name, and nothing of the bucket's list or writing
 		// (novox/hq ADR 0486 §2, issue 1697).
 		fileReads, err := DeclarationFilesReads(p.Node)

go/buscompose/state.go

@@ -325,3 +325,29 @@ func NodeComposeReads(node string) []string {
 	}
 	return append(out, readGrants(ResolvedBucket, node+".>")...)
 }
+
+// ServedReadGrants is the read of each key of mesh-config's resolved values named, `<node>.<module>.<key>`
+// (Principal.ServedReads, issue 1126): a get and a watch of that key alone. Each part is checked as every subject token
+// is, so a key cannot widen the grant.
+func ServedReadGrants(keys []string) ([]string, error) {
+	var out []string
+	seen := map[string]bool{}
+	for _, k := range keys {
+		parts := strings.Split(k, ".")
+		if len(parts) != 3 {
+			return nil, fmt.Errorf("%q is not <node>.<module>.<key>", k)
+		}
+		for _, part := range parts {
+			if !safeSubject.MatchString(part) {
+				return nil, fmt.Errorf("%q cannot be part of a subject: a permission is a subject pattern, and this would widen it", part)
+			}
+		}
+		for _, g := range readGrants(ResolvedBucket, k) {
+			if !seen[g] {
+				seen[g] = true
+				out = append(out, g)
+			}
+		}
+	}
+	return out, nil
+}

go/buscompose/users.go

@@ -5,6 +5,7 @@ import (
 	"git.novox.be/novox/mesh-sdk/go/announce"
 	"slices"
 	"sort"
+	"strings"
 )
 
 // Every user the composed file should contain, derived from what the mesh knows.
@@ -25,6 +26,9 @@ type Declared struct {
 	Emits    []string
 	Consumes []string
 	Serves   []string
+	// Served are the config keys its served values and zone name fill from its settings (catalogue.ServedSettings): what
+	// every node may read of its resolved values where it runs (Principal.ServedReads, issue 1126).
+	Served []string
 	// Holds are the seats this module claims, with the protocol each seat declares. A seat the
 	// mesh defines for itself declares no protocol, so holding one grants nothing on the bus —
 	// which is right: those seats are about who does a job, not about who may say what.
@@ -144,6 +148,7 @@ func Users(r Records) ([]Principal, error) {
 	out := []Principal{{Kind: KindController, Uses: asksSeatOf(r), AsksIssued: issuedOf(r), AsksTheVault: vaultVerbsOf(r),
 		AsksTheConfig: servedOf(r, VerbsTheControllerAsksTheConfig), FilesFor: sortedCopy(r.Nodes)}}
 
+	served := servedReadsOf(r)
 	for _, node := range sortedCopy(r.Nodes) {
 		witness := false
 		var checks []string
@@ -160,7 +165,7 @@ func Users(r Records) ([]Principal, error) {
 		out = append(out, Principal{Kind: KindNode, Node: node, WitnessesController: witness, Checks: checks,
 			MovesIntoTheVault: servesMoveIn(r), StatesItsKeyToTheVault: servesVerb(r, VaultNodeKey),
 			AsksItsOwnSecrets:     servesVerb(r, VaultOwn),
-			FollowsGrantsUserList: bus})
+			FollowsGrantsUserList: bus, ServedReads: servedOn(served, node)})
 		// **And the machine's provisioner client, on an account of its own** (novox/hq ADR 0349 clause 11): one
 		// per machine that runs it, delivered to the module running it.
 		for _, d := range r.Assigned[node] {
@@ -459,3 +464,32 @@ func declaredStreamsOf(r Records) []string {
 	sort.Strings(out)
 	return out
 }
+
+// servedReadsOf is every key of mesh-config's resolved values a module assigned on a node serves from,
+// `<node>.<module>.<key>`, sorted: each part a subject token, so a key that cannot be one is granted nothing (and the
+// node composing a consumer of it refuses, naming it), never a wider read.
+func servedReadsOf(r Records) []string {
+	var out []string
+	for node, declared := range r.Assigned {
+		for _, d := range declared {
+			for _, k := range d.Served {
+				if safeSubject.MatchString(node) && safeSubject.MatchString(d.Module) && safeSubject.MatchString(k) {
+					out = append(out, node+"."+d.Module+"."+k)
+				}
+			}
+		}
+	}
+	sort.Strings(out)
+	return slices.Compact(out)
+}
+
+// servedOn is, of the served keys, those of other nodes than node: its own slice it reads whole.
+func servedOn(served []string, node string) []string {
+	var out []string
+	for _, k := range served {
+		if !strings.HasPrefix(k, node+".") {
+			out = append(out, k)
+		}
+	}
+	return out
+}

go/catalogue/launchers.go

@@ -0,0 +1,25 @@
+package catalogue
+
+import "strings"
+
+// LauncherSuffix is what a TypeScript entrypoint's launcher is called beside it: index.js is started as
+// index.serve.mjs (novox/hq ADR 0193), an ES module by its own extension whatever the bundle's package.json says.
+const LauncherSuffix = ".serve.mjs"
+
+// Launchers are the launchers a bundle's build writes, by entrypoint, relative to the bundle's root: for TypeScript
+// every entrypoint `x.js` is started by `x.serve.mjs` beside it; an entrypoint that is not JavaScript and every other
+// language, whose build is already executable, get none. The one rule mesh-build-node, build-agent, the controller and
+// a node taking in a build share (R4e, issue 1126; buildstep.Launchers says it for the build modules), so the name a
+// build writes is the name a reader expects.
+func Launchers(language string, entrypoints []string) map[string]string {
+	out := map[string]string{}
+	if language != "typescript" {
+		return out
+	}
+	for _, entry := range entrypoints {
+		if strings.HasSuffix(entry, ".js") {
+			out[entry] = strings.TrimSuffix(entry, ".js") + LauncherSuffix
+		}
+	}
+	return out
+}

go/catalogue/settings.go

@@ -132,6 +132,30 @@ func Settle(base map[string]any, layers []Layer) (map[string]any, error) {
 	return filledFromSettings(deepCopy(base), layers, "what is served")
 }
 
+// ServedSettings is every key a module's served values and its zone's name fill from its settings (`${setting:<key>}`
+// in a top-level string, as Settle fills them), sorted: what every consumer of the module is told, wherever it runs,
+// and so the one part of a module's config values on its node that a node composing a consumer elsewhere reads (issue
+// 1126). None for a module that serves only literals.
+func ServedSettings(m Manifest) []string {
+	seen := map[string]bool{}
+	note := func(v any) {
+		if s, ok := v.(string); ok {
+			for _, k := range settingsUsed(s) {
+				seen[k] = true
+			}
+		}
+	}
+	for _, values := range m.Serves {
+		for _, v := range values {
+			note(v)
+		}
+	}
+	if m.Zone != nil {
+		note(m.Zone.Name)
+	}
+	return sortedKeys(seen)
+}
+
 // overridden lays settings over a contribution: a map whose keys are its contract. Only the keys the
 // map already declares are touched; the rest of a layer is somebody else's business (a file's,
 // another destination's) and is left to reach it there.

go/compose/frombus/frombus.go

@@ -14,10 +14,13 @@
 //   - what each node observes of itself (its tunnel key, capabilities, keys, certificate, links and networks): the
 //     node's own have key, `<node>.node` (HaveNode.Record); the tunnel found at a node's adoption and the names of the hub's peers are decided,
 //     in its placement, and whether the hub took the tunnel over and who enrolled under a peer's key are read from both;
-//   - every module's manifest: the indexer's `modules.<module>` and its manifests by digest, or the digest the
+//   - every module's manifest: the indexer's `modules.<module>` and its manifests by digest; for each module placed on
-//     composing node's want names where it runs another build (Scope.Kept);
+//     the composing node, its built manifest instead: the builder's done key at the trunk's FP2, or at the fingerprint
+//     the node's want names where it runs another build (Scope.KeptAt), resolved by the controller's own take-in rule
+//     (takein.Manifest), a moved module's templates archive read in the artifact store (issue 1126);
 //   - each module's config values on a node: mesh-config's resolved state, the composing node's slice alone unless the
-//     caller composes every node (ADR 0547 §2, the prepare step's L4).
+//     caller composes every node (ADR 0547 §2, the prepare step's L4); of another node, only the keys its modules'
+//     served values fill (catalogue.ServedSettings), which every consumer is told (issue 1126).
 //
 // **Own secrets and pair credentials** come from the vault sealed to the composing node's key (OwnSecrets, Credentials),
 // for that node alone: as a consumer and as a provider, exactly the keys compose.CredentialsNeeded names. **Not yet
@@ -32,6 +35,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/netip"
+	"slices"
 	"sort"
 	"strings"
 
@@ -40,6 +44,7 @@ import (
 	"git.novox.be/novox/mesh-sdk/go/declaration"
 	"git.novox.be/novox/mesh-sdk/go/delivery"
 	"git.novox.be/novox/mesh-sdk/go/overlay"
+	"git.novox.be/novox/mesh-sdk/go/takein"
 )
 
 // Reader is the bus as a caller reads it with its own credential. Each method reads one part's state; an error is a
@@ -64,6 +69,18 @@ type Reader interface {
 	// and each it grants as a provider, with the copy sealed to it (the vault's `own` with the keys, ADR 0550 §5). A key
 	// the vault holds no value for is left out; the composition then stands in for it, and Compose refuses it.
 	Credentials(ctx context.Context, node string, asConsumer, asProvider []compose.CredentialKey) ([]compose.Credential, []compose.Issued, error)
+	// Build is a module's build at a fingerprint, the builder's key (delivery.BuilderBuildsBucket, BuilderBuildKey); nil
+	// when the builder holds none. The composing node composes each of its modules from its build (issue 1126).
+	Build(ctx context.Context, module, fp string) (*delivery.BuilderBuild, error)
+	// Archive is the archive at url, a kept reference routed to the artifact store as the composing node reaches it
+	// (catalogue.Routed), checked against digest: a moved module's templates archive, read back for its texts as the
+	// controller's take-in reads it (takein.Built).
+	Archive(ctx context.Context, url, digest string) ([]byte, error)
+	// Served is, of one module's config values on another node than the composing one (mesh-config's state `resolved`,
+	// keys `<node>.<module>.<key>`), the keys named, decoded: the keys its served values and zone name fill
+	// (catalogue.ServedSettings), which every consumer is told and so every node may read (issue 1126). Nil when none
+	// is set.
+	Served(ctx context.Context, node, module string, keys []string) (map[string]any, error)
 }
 
 // Mesh is what is true of the whole mesh and is no part's state on the bus yet: the private network's range and the
@@ -79,8 +96,16 @@ type Mesh struct {
 }
 
 // Scope says who composes. Node is the node composing; only its config values are read unless Every is set, as the
-// composition check sets it to compose every node. Kept is, by module, the manifest digest the composing node's wants
+// composition check sets it to compose every node (of another node's modules, only the values their served values fill
-// name where it differs from the indexer's trunk: that node composes the module as it runs it.
+// are read, Reader.Served). KeptAt is, by module, the fingerprint the composing node's want names where it differs from
+// the indexer's trunk: that node composes the module from its build at that fingerprint, as it runs it.
+//
+// **Composed from its build** (issue 1126): each module placed on the composing node, and each in KeptAt, composes from
+// its built manifest (takein.Manifest over the builder's done key at the trunk's FP2 or the kept fingerprint), as the
+// controller composes from the manifest its take-in resolved: so a moved module is handed to the provisioner, a resource
+// naming an artifact names what was built, and a templated file has its text. A placed module the builder holds no done
+// build of at the trunk's FP2 composes from the indexer's manifest, as before. Every leaves the catalogue as the indexer
+// read it: the check of every node judges placement, and reads no archive.
 //
 // Placements, when set, stands in for the indexer's placement of the nodes it names: a node's candidate placement
 // (delivery.PlacementCandidate), so a node composes in shadow with a catalogue pull request's placement and compares
@@ -91,7 +116,7 @@ type Mesh struct {
 type Scope struct {
 	Node       string
 	Every      bool
-	Kept       map[string]string
+	KeptAt     map[string]string
 	Placements map[string]*delivery.Placement
 	InOwn      []string
 }
@@ -178,15 +203,18 @@ func Input(ctx context.Context, r Reader, mesh Mesh, s Scope) (compose.Input, er
 			}
 		}
 	}
+	parse := func(raw []byte) (catalogue.Manifest, error) {
+		if mesh.Predefined != nil {
+			return catalogue.ParseManifestWith(raw, mesh.Predefined)
+		}
+		return catalogue.ParseManifestForInputs(raw)
+	}
 	read := func(module, digest string) (catalogue.Manifest, error) {
 		raw, err := r.Manifest(ctx, digest)
 		if err != nil {
 			return catalogue.Manifest{}, fmt.Errorf("frombus: the manifest of %s (digest %s): %w", module, digest, err)
 		}
-		if mesh.Predefined != nil {
+		return parse(raw)
-			return catalogue.ParseManifestWith(raw, mesh.Predefined)
-		}
-		return catalogue.ParseManifestForInputs(raw)
 	}
 	for _, name := range sortedKeys(modules) {
 		m := modules[name]
@@ -230,6 +258,13 @@ func Input(ctx context.Context, r Reader, mesh Mesh, s Scope) (compose.Input, er
 				if values, err = r.Values(ctx, node, m.Module); err != nil {
 					return compose.Input{}, fmt.Errorf("frombus: %s's config values on %s: %w", m.Module, node, err)
 				}
+			} else if keys := catalogue.ServedSettings(in.Catalogue[m.Module]); len(keys) > 0 {
+				// What a provider elsewhere serves from its settings is told to every consumer: the composing node reads
+				// those keys of it, and no other (issue 1126: mailu's smtp domain, a consumer's on shanks).
+				if values, err = r.Served(ctx, node, m.Module, keys); err != nil {
+					return compose.Input{}, fmt.Errorf("frombus: what %s serves on %s (%s): %w", m.Module, node,
+						strings.Join(keys, ", "), err)
+				}
 			}
 			if layers := m.Layers(node, values); layers != nil {
 				if n.Settings == nil {
@@ -245,16 +280,6 @@ func Input(ctx context.Context, r Reader, mesh Mesh, s Scope) (compose.Input, er
 				}
 				n.InOwn[module] = true
 			}
-			for _, module := range sortedKeys(s.Kept) {
-				man, err := read(module, s.Kept[module])
-				if err != nil {
-					return compose.Input{}, err
-				}
-				if n.Kept == nil {
-					n.Kept = map[string]catalogue.Manifest{}
-				}
-				n.Kept[module] = man
-			}
 		}
 		in.Nodes = append(in.Nodes, n)
 		in.Holdings = append(in.Holdings, p.Held()...)
@@ -284,6 +309,9 @@ func Input(ctx context.Context, r Reader, mesh Mesh, s Scope) (compose.Input, er
 		if _, ok := records[s.Node]; !ok {
 			return compose.Input{}, fmt.Errorf("frombus: %s has written no record of itself yet", s.Node)
 		}
+		if err := fromBuilds(ctx, r, &in, s, placements[s.Node], modules, read, parse); err != nil {
+			return compose.Input{}, err
+		}
 		// Its own modules' secrets, sealed to its own key by the vault: a composition carries them sealed (ADR 0550 §5).
 		// Asked only when a module placed on it declares one: a node none of whose modules has a secret composes
 		// whether or not the vault answers.
@@ -325,6 +353,99 @@ func Compose(ctx context.Context, r Reader, mesh Mesh, s Scope, fps map[string]s
 	return compose.CompositionsApplied(in, s.Node, fps)
 }
 
+// fromBuilds puts in the built manifest of each module placed on the composing node, at the trunk's FP2, and of each
+// module it keeps at another build (Scope.KeptAt, into its node's Kept), read from the builder's done key and resolved
+// by the one take-in rule (takein.Manifest), a templates archive read in the artifact store as the node reaches it
+// (issue 1126).
+func fromBuilds(ctx context.Context, r Reader, in *compose.Input, s Scope, p delivery.Placement, modules map[string]delivery.Module,
+	read func(module, digest string) (catalogue.Manifest, error), parse func([]byte) (catalogue.Manifest, error)) error {
+	address, addressErr := "", error(nil)
+	addressed := false
+	archive := func(ctx context.Context, reference, digest string) ([]byte, error) {
+		if !addressed {
+			address, addressErr = compose.ArtifactStoreAddress(*in, s.Node)
+			addressed = true
+		}
+		switch {
+		case addressErr != nil:
+			return nil, addressErr
+		case address == "":
+			return nil, fmt.Errorf("%s reaches no artifact store", s.Node)
+		}
+		return r.Archive(ctx, catalogue.Routed(reference, address), digest)
+	}
+	built := func(module, fp string, b *delivery.BuilderBuild) (catalogue.Manifest, error) {
+		if b.Manifest == "" {
+			return catalogue.Manifest{}, fmt.Errorf("frombus: the build of %s at %s names no manifest", module, fp)
+		}
+		src, err := read(module, b.Manifest)
+		if err != nil {
+			return catalogue.Manifest{}, err
+		}
+		resolved, err := takein.Manifest(ctx, src, *b, archive)
+		if err != nil {
+			return catalogue.Manifest{}, fmt.Errorf("frombus: %s at %s: %w", module, fp, err)
+		}
+		// Read back as the mesh holds a built manifest (the controller keeps the resolved manifest's bytes and reads them
+		// as any manifest): a template's text is then in the manifest's own words, its ${config:…} in the settings form.
+		raw, err := json.Marshal(resolved)
+		if err != nil {
+			return catalogue.Manifest{}, err
+		}
+		if resolved, err = parse(raw); err != nil {
+			return catalogue.Manifest{}, fmt.Errorf("frombus: the built manifest of %s at %s: %w", module, fp, err)
+		}
+		return resolved, nil
+	}
+	for _, pm := range p.Modules {
+		m, ok := modules[pm.Module]
+		if !ok || m.FP2 == "" || s.KeptAt[pm.Module] != "" || slices.Contains(s.InOwn, pm.Module) {
+			// A module it runs inside its own composition is composed there as before: no build of the new form is what
+			// it runs (Scope.InOwn).
+			continue
+		}
+		if man, ok := in.Catalogue[pm.Module]; !ok || (man.Build == nil && !man.Templated()) {
+			continue // nothing is built of it: its manifest is what it composes from
+		}
+		b, err := r.Build(ctx, pm.Module, m.FP2)
+		if err != nil {
+			return fmt.Errorf("frombus: the build of %s at %s: %w", pm.Module, m.FP2, err)
+		}
+		if b == nil || b.State != delivery.BuildDone {
+			continue // no done build at the trunk's: composed from the indexer's manifest, as before
+		}
+		resolved, err := built(pm.Module, m.FP2, b)
+		if err != nil {
+			return err
+		}
+		in.Catalogue[pm.Module] = resolved
+	}
+	for i := range in.Nodes {
+		if in.Nodes[i].Name != s.Node {
+			continue
+		}
+		for _, module := range sortedKeys(s.KeptAt) {
+			fp := s.KeptAt[module]
+			b, err := r.Build(ctx, module, fp)
+			if err != nil {
+				return fmt.Errorf("frombus: the build of %s at %s: %w", module, fp, err)
+			}
+			if b == nil || b.State != delivery.BuildDone {
+				return fmt.Errorf("frombus: %s is wanted at %s, of which the builder holds no done build", module, fp)
+			}
+			resolved, err := built(module, fp, b)
+			if err != nil {
+				return err
+			}
+			if in.Nodes[i].Kept == nil {
+				in.Nodes[i].Kept = map[string]catalogue.Manifest{}
+			}
+			in.Nodes[i].Kept[module] = resolved
+		}
+	}
+	return nil
+}
+
 // declaresOwnSecrets says a module assigned to node declares an own secret, in the manifest the node composes it from.
 func declaresOwnSecrets(in compose.Input, node string) bool {
 	for _, n := range in.Nodes {

go/compose/frombus/frombus_test.go

@@ -1,6 +1,9 @@
 package frombus
 
 import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
 	"context"
 	"crypto/ecdh"
 	"crypto/rand"
@@ -10,6 +13,7 @@ import (
 	"fmt"
 	"os"
 	"reflect"
+	"slices"
 	"strings"
 	"testing"
 
@@ -32,6 +36,39 @@ type bus struct {
 	// asked is every consumer credential asked; the fake refuses the whole ask, as the vault's own does, for a consuming
 	// module its placement does not put on the node.
 	asked []compose.CredentialKey
+	// builds are the builder's keys, by "<module> <fp>"; archives the artifact store's blobs, by digest, and fetched
+	// every URL asked; served another node's values by "<node>.<module>", and servedRead every ask with its keys.
+	builds     map[string]*delivery.BuilderBuild
+	archives   map[string][]byte
+	fetched    []string
+	served     map[string]map[string]any
+	servedRead []string
+	buildsRead []string
+}
+
+func (b *bus) Build(_ context.Context, module, fp string) (*delivery.BuilderBuild, error) {
+	b.buildsRead = append(b.buildsRead, module)
+	return b.builds[module+" "+fp], nil
+}
+
+func (b *bus) Archive(_ context.Context, url, digest string) ([]byte, error) {
+	b.fetched = append(b.fetched, url)
+	body, ok := b.archives[digest]
+	if !ok {
+		return nil, errors.New("the store holds no such blob")
+	}
+	return body, nil
+}
+
+func (b *bus) Served(_ context.Context, node, module string, keys []string) (map[string]any, error) {
+	b.servedRead = append(b.servedRead, node+"."+module+":"+strings.Join(keys, ","))
+	out := map[string]any{}
+	for _, k := range keys {
+		if v, ok := b.served[node+"."+module][k]; ok {
+			out[k] = v
+		}
+	}
+	return out, nil
 }
 
 func (b *bus) Placements(context.Context) (map[string]delivery.Placement, error) {
@@ -442,11 +479,193 @@ func TestInput(t *testing.T) {
 		t.Error("laptop's media, which declares an own secret, composed without the vault")
 	}
 
+	// A module the node's want keeps at another build than the trunk's composes from that build's manifest, resolved with
+	// what it made (issue 1126), on the composing node alone.
 	kept := mk()
-	kept.manifests["old"] = xterm
+	oldXterm := []byte(`{"module":"xterm","version":"1","resources":[{"id":"tools","type":"process","name":"xterm-tools",` +
-	in, err := Input(ctx, kept, Mesh{}, Scope{Node: "laptop", Kept: map[string]string{"xterm": "old"}})
+		`"artifact":"tools","run":["./xterm-tools"]}],"build":{"artifacts":[{"name":"tools","kind":"bundle","language":"go","system":"arch","from":"cmd/xterm-tools","binary":"xterm-tools"}]}}`)
-	if err != nil || !reflect.DeepEqual(in.Nodes[1].Kept, map[string]catalogue.Manifest{"xterm": parsed}) || in.Nodes[0].Kept != nil {
+	kept.manifests["old"] = oldXterm
-		t.Errorf("the want's manifest is kept on the composing node alone: %v, %v", in.Nodes, err)
+	oldFP := "src2:" + strings.Repeat("9", 64)
+	toolsRef := "artifact-store://xterm/tools/blobs/sha256:" + strings.Repeat("1", 64)
+	kept.builds = map[string]*delivery.BuilderBuild{"xterm " + oldFP: {State: delivery.BuildDone, Manifest: "old",
+		Artifacts: []delivery.BuiltArtifact{{Name: "tools", Kind: "bundle", Reference: toolsRef, Digest: "sha256:" + strings.Repeat("1", 64)}}}}
+	parsedOld, err := catalogue.ParseManifestForInputs(oldXterm)
+	if err != nil {
+		t.Fatal(err)
+	}
+	keptBuilt, err := parsedOld.Resolve([]catalogue.Built{{Name: "tools", Kind: "bundle", Reference: toolsRef, Digest: "sha256:" + strings.Repeat("1", 64)}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	in, err := Input(ctx, kept, Mesh{}, Scope{Node: "laptop", KeptAt: map[string]string{"xterm": oldFP}})
+	if err != nil || !reflect.DeepEqual(in.Nodes[1].Kept, map[string]catalogue.Manifest{"xterm": keptBuilt}) || in.Nodes[0].Kept != nil {
+		t.Errorf("the want's build is kept on the composing node alone: %v, %v", in.Nodes, err)
+	}
+
+	// Issue 1126, the seed's refusals. restic and the provisioner's client as the catalogue holds them (mesh-catalog
+	// c05aa5e6), restic moved with an own secret the vault leaves out (it hands it to the client, mesh-vault ownnode.go
+	// handed): composed from the indexer's manifest restic is not handed and stands in for that secret, as novox's,
+	// ace's, g14's and shanks's answers refused; composed from its build, it is handed, its templates archive read
+	// where the node reaches the artifact store, and the composition is the one the controller's take-in gives.
+	resticRaw, err := os.ReadFile("testdata/catalogue-c05aa5e6-restic.module.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	clientRaw, err := os.ReadFile("testdata/catalogue-c05aa5e6-mesh-provisioner-client.module.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	store := []byte(`{"module":"store","version":"1","provides":[{"name":"artifact-store","scope":"mesh"}],` +
+		`"serves":{"artifact-store":{"port":5000}}}`)
+	texts := map[string]string{}
+	var packed bytes.Buffer
+	zw := gzip.NewWriter(&packed)
+	tw := tar.NewWriter(zw)
+	for _, f := range []string{"backups.conf", "data.conf", "retention.conf"} {
+		text, err := os.ReadFile("testdata/catalogue-c05aa5e6-restic.files/" + f)
+		if err != nil {
+			t.Fatal(err)
+		}
+		texts[f] = string(text)
+		if err := tw.WriteHeader(&tar.Header{Name: "./" + f, Mode: 0o644, Size: int64(len(text)), Typeflag: tar.TypeReg}); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := tw.Write(text); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := tw.Close(); err != nil {
+		t.Fatal(err)
+	}
+	if err := zw.Close(); err != nil {
+		t.Fatal(err)
+	}
+	archiveDigest := delivery.ManifestDigest(packed.Bytes())
+	digestOf := func(c string) string { return "sha256:" + strings.Repeat(c, 64) }
+	movedFps := map[string]string{"restic": "src2:" + strings.Repeat("3", 64), "mesh-provisioner-client": "src2:" + strings.Repeat("4", 64),
+		"store": "src2:" + strings.Repeat("5", 64)}
+	resticArtifacts := []delivery.BuiltArtifact{{Name: "tools", Kind: "bundle", Reference: "artifact-store://restic/tools/blobs/" + digestOf("6"), Digest: digestOf("6")},
+		{Name: "files", Kind: "archive", Reference: "artifact-store://restic/files/blobs/" + archiveDigest, Digest: archiveDigest}}
+	clientArtifacts := []delivery.BuiltArtifact{{Name: "client", Kind: "bundle", Reference: "artifact-store://mesh-provisioner-client/client/blobs/" + digestOf("7"), Digest: digestOf("7")}}
+	movedMesh := func(built bool) *bus {
+		b := one()
+		p := b.placements["n1"]
+		p.Modules = []delivery.PlacedModule{{Module: "store"}, {Module: "mesh-provisioner-client"}, {Module: "restic"}}
+		p.Decided.Account, p.Decided.AccountHome = "operator", "/home/operator"
+		b.placements["n1"] = p
+		rec := b.records["n1"]
+		rec.Contract, rec.Capabilities = compose.HandedContract, []string{"systemd", "package-manager"}
+		b.records["n1"] = rec
+		b.modules = map[string]delivery.Module{}
+		b.manifests = map[string][]byte{}
+		for module, raw := range map[string][]byte{"restic": resticRaw, "mesh-provisioner-client": clientRaw, "store": store} {
+			b.modules[module] = delivery.Module{FP2: movedFps[module], Manifest: delivery.ManifestDigest(raw)}
+			b.manifests[delivery.ManifestDigest(raw)] = raw
+		}
+		// The vault's own answer: the client's own secret; restic's is the client's to ask, so own leaves it out.
+		b.own = []compose.OwnSecretRow{{Node: "n1", Module: "mesh-provisioner-client", Name: "provisioner-broker", Origin: "made", SealedTo: "sk", Sealed: `"box"`}}
+		if built {
+			b.builds = map[string]*delivery.BuilderBuild{
+				"restic " + movedFps["restic"]:                                   {State: delivery.BuildDone, Manifest: delivery.ManifestDigest(resticRaw), Artifacts: resticArtifacts},
+				"mesh-provisioner-client " + movedFps["mesh-provisioner-client"]: {State: delivery.BuildDone, Manifest: delivery.ManifestDigest(clientRaw), Artifacts: clientArtifacts},
+				"store " + movedFps["store"]:                                     {State: delivery.BuildDone, Manifest: delivery.ManifestDigest(store)},
+			}
+			b.archives = map[string][]byte{archiveDigest: packed.Bytes()}
+		}
+		return b
+	}
+	if _, _, err := Compose(ctx, movedMesh(false), Mesh{}, Scope{Node: "n1"}, movedFps); err == nil || !strings.Contains(err.Error(), `restic needs a secret called "repository"`) {
+		t.Errorf("restic composed from the indexer's manifest: %v, want its own secret refused", err)
+	}
+	builtBus := movedMesh(true)
+	fromBuild, _, err := Compose(ctx, builtBus, Mesh{}, Scope{Node: "n1"}, movedFps)
+	if err != nil {
+		t.Fatalf("restic composed from its build: %v", err)
+	}
+	if !reflect.DeepEqual(builtBus.fetched, []string{"http://127.0.0.1:5000/v2/restic/files/blobs/" + archiveDigest}) {
+		t.Errorf("the templates archive was read at %v", builtBus.fetched)
+	}
+	resticKey, _ := delivery.ComposedKey("n1", "restic", movedFps["restic"])
+	if c := fromBuild[resticKey]; len(c.Filled) != 4 { // its three templated files and its own secret
+		t.Errorf("restic is not handed to the provisioner: its filled files are %v", c.Filled)
+	}
+	// A done key an artifact of which names no reference is waited for, in the controller's words (ADR 0498 §8).
+	unreferenced := movedMesh(true)
+	unreferenced.builds["restic "+movedFps["restic"]].Artifacts = []delivery.BuiltArtifact{{Name: "tools", Kind: "bundle", Digest: digestOf("6")}, resticArtifacts[1]}
+	if _, _, err := Compose(ctx, unreferenced, Mesh{}, Scope{Node: "n1"}, movedFps); err == nil || !strings.Contains(err.Error(), "the build of restic is done and waits for the reference of tools") {
+		t.Errorf("a done key with an unreferenced artifact: %v", err)
+	}
+	// A module the node runs inside its own composition (the fold) composes from the indexer's manifest though a done
+	// build of it is at the trunk's FP2: its build is not read.
+	folding := movedMesh(true)
+	foldIn, err := Input(ctx, folding, Mesh{}, Scope{Node: "n1", InOwn: []string{"mesh-provisioner-client"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	clientParsed, err := catalogue.ParseManifestForInputs(clientRaw)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if slices.Contains(folding.buildsRead, "mesh-provisioner-client") || !reflect.DeepEqual(foldIn.Catalogue["mesh-provisioner-client"], clientParsed) {
+		t.Errorf("a folded module was taken from its build: builds read %v", folding.buildsRead)
+	}
+	// The controller's: its take-in resolves each manifest with the key's artifacts and the archive's texts, and composes
+	// from those over the same input.
+	storeIn, err := Input(ctx, movedMesh(false), Mesh{}, Scope{Node: "n1"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for module, built := range map[string][]catalogue.Built{
+		"restic": {{Name: "tools", Kind: "bundle", Reference: resticArtifacts[0].Reference, Digest: digestOf("6")},
+			{Name: "files", Kind: "archive", Reference: resticArtifacts[1].Reference, Digest: archiveDigest, Texts: texts}},
+		"mesh-provisioner-client": {{Name: "client", Kind: "bundle", Reference: clientArtifacts[0].Reference, Digest: digestOf("7")}},
+	} {
+		resolved, err := storeIn.Catalogue[module].Resolve(built)
+		if err != nil {
+			t.Fatal(err)
+		}
+		// Kept as its bytes and read again, as the controller's store holds a build's manifest.
+		raw, _ := json.Marshal(resolved)
+		if storeIn.Catalogue[module], err = catalogue.ParseManifestForInputs(raw); err != nil {
+			t.Fatal(err)
+		}
+	}
+	storeIn.OwnSecrets = movedMesh(true).own
+	fromStoreMoved, _, err := compose.Compositions(storeIn, "n1", movedFps, compose.Reading)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(fromBuild, fromStoreMoved) {
+		gj, _ := json.MarshalIndent(fromBuild, "", " ")
+		wj, _ := json.MarshalIndent(fromStoreMoved, "", " ")
+		t.Errorf("composed from the build differs from the controller's take-in:\n got %s\nwant %s", gj, wj)
+	}
+
+	// And what a provider on another node serves from its settings (mailu's smtp domain, issue 1126: shanks refused
+	// '${setting:domain} ... nothing sets "domain"'): read for that provider's served keys alone, through Served.
+	mailer := []byte(`{"module":"mailer","version":"1","config":{"domain":{"type":"string","description":"the mail domain","required":true}},` +
+		`"provides":[{"name":"smtp","scope":"mesh"}],"serves":{"smtp":{"port":587,"domain":"${config:domain}"}}}`)
+	withMailer := func(set bool) *bus {
+		b := mk()
+		p := b.placements["anchor"]
+		p.Modules = append(p.Modules, delivery.PlacedModule{Module: "mailer"})
+		b.placements["anchor"] = p
+		b.modules["mailer"] = delivery.Module{Manifest: delivery.ManifestDigest(mailer)}
+		b.manifests[delivery.ManifestDigest(mailer)] = mailer
+		if set {
+			b.served = map[string]map[string]any{"anchor.mailer": {"domain": "example.org"}}
+		}
+		return b
+	}
+	if _, _, err := Compose(ctx, withMailer(false), Mesh{}, Scope{Node: "laptop"}, fps); err == nil || !strings.Contains(err.Error(), `nothing sets "domain"`) {
+		t.Errorf("a provider's served setting nobody sets: %v", err)
+	}
+	served := withMailer(true)
+	if _, _, err := Compose(ctx, served, Mesh{}, Scope{Node: "laptop"}, fps); err != nil {
+		t.Errorf("laptop with what anchor's mailer serves: %v", err)
+	}
+	if !reflect.DeepEqual(served.servedRead, []string{"anchor.mailer:domain"}) || slices.Contains(served.valuesRead, "anchor.mailer") {
+		t.Errorf("laptop read %v of other nodes (and %v of config values): the served keys of anchor's mailer alone", served.servedRead, served.valuesRead)
 	}
 
 	for name, c := range map[string]struct {
@@ -457,7 +676,7 @@ func TestInput(t *testing.T) {
 		"no record":          {func(b *bus) { delete(b.records, "laptop") }, Scope{Node: "anchor"}, "laptop, whose agent has written no record"},
 		"no placement":       {func(b *bus) {}, Scope{Node: "stranger"}, "stranger has no placement"},
 		"a manifest unread":  {func(b *bus) { delete(b.manifests, digest) }, Scope{Node: "laptop"}, "the manifest of xterm"},
-		"a kept one unread": {func(b *bus) {}, Scope{Node: "laptop", Kept: map[string]string{"xterm": "gone"}}, "digest gone"},
+		"a kept one unbuilt": {func(b *bus) {}, Scope{Node: "laptop", KeptAt: map[string]string{"xterm": "src2:gone"}}, "holds no done build"},
 	} {
 		b := mk()
 		c.change(b)

go/compose/frombus/testdata/catalogue-c05aa5e6-mesh-provisioner-client.module.json

@@ -0,0 +1,60 @@
+{
+  "module": "mesh-provisioner-client",
+  "version": "1",
+  "claims": [
+    {
+      "name": "node-provisioner",
+      "scope": "node"
+    }
+  ],
+  "own-secrets": {
+    "provisioner-broker": "${dir:state}/broker"
+  },
+  "secrets-owner": "mesh-provisioner-client",
+  "resources": [
+    {
+      "id": "account",
+      "type": "user",
+      "name": "mesh-provisioner-client",
+      "shell": "/usr/bin/nologin",
+      "home": "/var/lib/mesh-provisioner-client"
+    },
+    {
+      "id": "state",
+      "type": "directory",
+      "mode": "0700",
+      "owner": "mesh-provisioner-client",
+      "place": ".",
+      "unknown": "program"
+    },
+    {
+      "id": "client",
+      "type": "process",
+      "name": "mesh-provisioner-client",
+      "artifact": "client",
+      "run": [
+        "./mesh-provisioner-client"
+      ],
+      "env": {
+        "MESH_PROVISIONER_CREDENTIAL": "${dir:state}/broker",
+        "MESH_PROVISIONER_STATE": "${dir:state}"
+      },
+      "user": "mesh-provisioner-client",
+      "health": {
+        "kind": "unit"
+      }
+    }
+  ],
+  "build": {
+    "artifacts": [
+      {
+        "name": "client",
+        "kind": "bundle",
+        "language": "go",
+        "system": "arch",
+        "from": "cmd/mesh-provisioner-client",
+        "binary": "mesh-provisioner-client"
+      }
+    ]
+  }
+}

go/compose/frombus/testdata/catalogue-c05aa5e6-restic.files/backups.conf

@@ -0,0 +1,2 @@
+# What the modules on this machine back up, composed by the mesh (novox/hq to-be 43). Do not edit.
+${backup:lines}

go/compose/frombus/testdata/catalogue-c05aa5e6-restic.files/data.conf

@@ -0,0 +1,2 @@
+# The data the modules on this machine declare, composed by the mesh (novox/hq ADR 0233). Do not edit.
+${backup:data}

go/compose/frombus/testdata/catalogue-c05aa5e6-restic.files/retention.conf

@@ -0,0 +1,6 @@
+# How many restore points the night keeps, written by the mesh from this machine's settings keep-daily,
+# keep-weekly, keep-monthly (novox/hq issue 416) and keep-hand-taken-days (novox/hq issue 453). Replaced on every push; change the settings, never this file.
+daily=${config:keep-daily}
+weekly=${config:keep-weekly}
+monthly=${config:keep-monthly}
+hand=${config:keep-hand-taken-days}

go/compose/frombus/testdata/catalogue-c05aa5e6-restic.module.json

@@ -0,0 +1,199 @@
+{
+  "module": "restic",
+  "version": "1",
+  "moved": true,
+  "claims": [
+    {
+      "name": "node-backup",
+      "scope": "node",
+      "serves": [
+        "backed-up",
+        "now",
+        "restore"
+      ]
+    }
+  ],
+  "emits": [
+    "backed-up"
+  ],
+  "tools": [
+    "backup_health",
+    "backup_delete_retired",
+    "backup_deleted"
+  ],
+  "own-secrets": {
+    "repository": "${dir:state}/repository.secret"
+  },
+  "data": {
+    "own": [
+      {
+        "id": "repository",
+        "path": "${dir:repository}",
+        "class": "rebuildable",
+        "backup": "none",
+        "why": "the restore points themselves: a copy of data kept elsewhere on this machine, never the only copy of anything; lost, the history goes and nothing live does"
+      },
+      {
+        "id": "state",
+        "path": "${dir:state}",
+        "class": "cache",
+        "why": "what the last nights and measurements were; the next night writes it again"
+      }
+    ]
+  },
+    "config": {
+      "keep-daily": {
+        "type": "number",
+        "description": "How many nightly restore points are kept, a count from 1 to 9999: two weeks undo a mistake noticed within a fortnight. Read by the night, which thins nothing while it does not read (novox/hq issue 416); lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.",
+        "minimum": 1,
+        "maximum": 9999,
+        "trusted": true,
+        "default": 14
+      },
+      "keep-weekly": {
+        "type": "number",
+        "description": "How many weekly restore points are kept behind the dailies, a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.",
+        "minimum": 1,
+        "maximum": 9999,
+        "trusted": true,
+        "default": 8
+      },
+      "keep-monthly": {
+        "type": "number",
+        "description": "How many monthly restore points are kept behind the weeklies, a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.",
+        "minimum": 1,
+        "maximum": 9999,
+        "trusted": true,
+        "default": 6
+      },
+      "keep-hand-taken-days": {
+        "type": "number",
+        "description": "How many days a restore point taken by hand (node-backup.now, or the mesh's before a module moves) is kept at least, a count from 1 to 9999; it is kept until a later good night of its module exists, and this many days in any case, so the next thinning never takes the one taken against a risky change (novox/hq issue 453). Lowering it lets go of the older ones taken by hand at the next thinning, and that cannot be undone.",
+        "minimum": 1,
+        "maximum": 9999,
+        "trusted": true,
+        "default": 7
+      }
+    },
+  "resources": [
+    {
+      "id": "state",
+      "type": "directory",
+      "mode": "0700",
+      "place": ".",
+      "unknown": "program"
+    },
+    {
+      "id": "repository",
+      "type": "directory",
+      "mode": "0700",
+      "unknown": "program"
+    },
+    {
+      "id": "declared",
+      "type": "file",
+      "path": "${dir:state}/backups.conf",
+      "mode": "0600",
+      "template": "files/backups.conf"
+    },
+    {
+      "id": "data-declared",
+      "type": "file",
+      "path": "${dir:state}/data.conf",
+      "mode": "0600",
+      "template": "files/data.conf"
+    },
+    {
+      "id": "retention",
+      "type": "file",
+      "path": "${dir:state}/retention.conf",
+      "mode": "0644",
+      "trusted": true,
+      "template": "files/retention.conf"
+    },
+    {
+      "id": "tool",
+      "type": "package",
+      "package": "restic"
+    },
+    {
+      "id": "sqlite",
+      "type": "package",
+      "package": "sqlite"
+    },
+    {
+      "id": "night",
+      "type": "process",
+      "name": "restic-night",
+      "artifact": "tools",
+      "run": [
+        "./restic-backups",
+        "night"
+      ],
+      "user": "${machine:account}",
+      "env": {
+        "MESH_BACKUP_DECLARED": "${dir:state}/backups.conf",
+        "MESH_BACKUP_REPOSITORY": "${dir:repository}",
+        "MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret",
+        "MESH_BACKUP_STATE": "${dir:state}",
+        "MESH_BACKUP_DATA": "${dir:state}/data.conf"
+      },
+      "health": {
+        "kind": "tool",
+        "tool": "backup_health",
+        "interval": "60s",
+        "timeout": "20s",
+        "looks": 2,
+        "grace": "60s"
+      }
+    },
+    {
+      "id": "measure",
+      "type": "process",
+      "name": "restic-measure",
+      "artifact": "tools",
+      "run": [
+        "./restic-backups",
+        "measure"
+      ],
+      "user": "${machine:account}",
+      "env": {
+        "MESH_BACKUP_DECLARED": "${dir:state}/backups.conf",
+        "MESH_BACKUP_REPOSITORY": "${dir:repository}",
+        "MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret",
+        "MESH_BACKUP_STATE": "${dir:state}",
+        "MESH_BACKUP_DATA": "${dir:state}/data.conf"
+      },
+      "health": {
+        "kind": "unit"
+      }
+    }
+  ],
+  "build": {
+    "artifacts": [
+      {
+        "name": "tools",
+        "kind": "bundle",
+        "language": "go",
+        "system": "arch",
+        "from": "cmd/restic-backups",
+        "binary": "restic-backups",
+        "loads": [
+          "restic-backups"
+        ],
+        "env": {
+          "MESH_BACKUP_DECLARED": "${dir:state}/backups.conf",
+          "MESH_BACKUP_REPOSITORY": "${dir:repository}",
+          "MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret",
+          "MESH_BACKUP_STATE": "${dir:state}",
+          "MESH_BACKUP_DATA": "${dir:state}/data.conf"
+        }
+      },
+      {
+        "name": "files",
+        "kind": "archive",
+        "from": "files"
+      }
+    ]
+  }
+}

go/compose/rendering.go

@@ -325,6 +325,13 @@ func (c *composer) artifactStoreOnNetwork(on map[string]bool) (node, port string
 	return "", "", false, nil
 }
 
+// ArtifactStoreAddress is the artifact store as node reaches it, by the rule its composition routes every kept reference
+// with (artifactStoreAddress): empty when node reaches none. A node composing its own modules reads a templates archive
+// there (issue 1126).
+func ArtifactStoreAddress(in Input, node string) (string, error) {
+	return newComposer(in).artifactStoreAddress(node)
+}
+
 // artifactStoreAddress is the artifact store as `forNode` reaches it: `<node>.internal:<port>` over the private network, or
 // `127.0.0.1:<port>` for the node that holds the store itself when nothing is on the network yet, and empty for any other
 // (novox/hq 04-ISSUES/102).

go/takein/takein.go

@@ -0,0 +1,134 @@
+// Package takein is the one rule that turns a builder's done key into the manifest the mesh composes a module from
+// (issue 1126): the module's manifest resolved with what its build made. It is mesh-controller's take-in of a done key
+// (cmd/mesh-controller/key_takein.go builtOfKey, unpackTexts and Manifest.Resolve) moved into the SDK, so a node agent
+// composing its own modules (novox/hq ADR 0547 §2) composes from the same built manifest the controller composes from:
+// every artifact a resource names resolved to its reference and digest, a TypeScript bundle's launchers, and, for a
+// moved module, each template's text read back from its templates archive and what its build keeps for the
+// provisioner (catalogue.Handed). Composing from the manifest the indexer read instead hands no moved module and
+// leaves every artifact unbuilt.
+//
+// It reads nothing itself: the archive is read through the caller's Archive, from wherever the caller reaches the
+// artifact store.
+package takein
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"strings"
+
+	"git.novox.be/novox/mesh-sdk/go/catalogue"
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+)
+
+// Archive reads the archive a recorded reference names (catalogue.Recorded: the artifact store's address left out),
+// checked against its digest.
+type Archive func(ctx context.Context, reference, digest string) ([]byte, error)
+
+// Manifest is m, as its manifest at b's fingerprint reads, resolved with what b made: the built manifest. b must be done
+// and every artifact must name its reference.
+// A templated module's templates archive is read through archive; a module without templates reads none, and archive
+// may then be nil.
+func Manifest(ctx context.Context, m catalogue.Manifest, b delivery.BuilderBuild, archive Archive) (catalogue.Manifest, error) {
+	if b.State != delivery.BuildDone {
+		return catalogue.Manifest{}, fmt.Errorf("takein: the build of %s is %s, not done: nothing to resolve it with", m.Module, b.State)
+	}
+	// **A done key whose artifact names no reference is waited for, never composed from** (novox/hq ADR 0498 §8), in the
+	// controller's words (mesh-controller internal/builderkeys Key.Says): a key done before R4e, or a candidate's image or
+	// package not yet built again for the trunk.
+	var missing []string
+	for _, a := range b.Artifacts {
+		if strings.TrimSpace(a.Reference) == "" {
+			missing = append(missing, a.Name)
+		}
+	}
+	if len(missing) > 0 {
+		return catalogue.Manifest{}, fmt.Errorf("takein: the build of %s is done and waits for the reference of %s",
+			m.Module, strings.Join(missing, ", "))
+	}
+	built, err := Built(ctx, m, b.Artifacts, archive)
+	if err != nil {
+		return catalogue.Manifest{}, err
+	}
+	resolved, err := m.Resolve(built)
+	if err != nil {
+		return catalogue.Manifest{}, fmt.Errorf("takein: the build of %s does not resolve: %w", m.Module, err)
+	}
+	return resolved, nil
+}
+
+// Built is each of a done key's artifacts as catalogue.Manifest.Resolve reads it: its recorded reference and digest, a
+// TypeScript bundle's launchers by the build modules' one rule (catalogue.Launchers), and, for the archive a module packs
+// its templates into, each template's text read back from that archive.
+func Built(ctx context.Context, m catalogue.Manifest, artifacts []delivery.BuiltArtifact, archive Archive) ([]catalogue.Built, error) {
+	declared := map[string]catalogue.Artifact{}
+	if m.Build != nil {
+		for _, a := range m.Build.Artifacts {
+			declared[a.Name] = a
+		}
+	}
+	templates := ""
+	if m.Templated() {
+		templates = catalogue.TemplatesArtifact(m)
+	}
+	var out []catalogue.Built
+	for _, a := range artifacts {
+		d := declared[a.Name]
+		kind := a.Kind
+		if kind == "" {
+			kind = d.Kind
+		}
+		b := catalogue.Built{Name: a.Name, Kind: kind, Reference: catalogue.Recorded(a.Reference), Digest: a.Digest}
+		if launchers := catalogue.Launchers(d.Language, d.Entrypoints); len(launchers) > 0 {
+			b.Launchers = launchers
+		}
+		if a.Name == templates && templates != "" {
+			if archive == nil {
+				return nil, fmt.Errorf("takein: %s packs its templates into %q, and nothing here reads the artifact store",
+					m.Module, a.Name)
+			}
+			body, err := archive(ctx, b.Reference, b.Digest)
+			if err != nil {
+				return nil, fmt.Errorf("takein: the templates archive %q of %s cannot be read from the artifact store: %w",
+					a.Name, m.Module, err)
+			}
+			if b.Texts, err = UnpackTexts(body); err != nil {
+				return nil, fmt.Errorf("takein: the templates archive %q of %s cannot be read back: %w", a.Name, m.Module, err)
+			}
+		}
+		out = append(out, b)
+	}
+	return out, nil
+}
+
+// UnpackTexts is every regular file of a packed templates archive (gzip over tar), by its path inside it.
+func UnpackTexts(body []byte) (map[string]string, error) {
+	zipped, err := gzip.NewReader(bytes.NewReader(body))
+	if err != nil {
+		return nil, err
+	}
+	defer zipped.Close()
+	reader := tar.NewReader(zipped)
+	texts := map[string]string{}
+	for {
+		header, err := reader.Next()
+		if errors.Is(err, io.EOF) {
+			return texts, nil
+		}
+		if err != nil {
+			return nil, err
+		}
+		if header.Typeflag != tar.TypeReg {
+			continue
+		}
+		text, err := io.ReadAll(io.LimitReader(reader, 16<<20))
+		if err != nil {
+			return nil, err
+		}
+		texts[strings.TrimPrefix(header.Name, "./")] = string(text)
+	}
+}