diff --git a/go/buildstep/launchers.go b/go/buildstep/launchers.go index 4b59f5c..975f560 100644 --- a/go/buildstep/launchers.go +++ b/go/buildstep/launchers.go @@ -1,24 +1,14 @@ package buildstep -import "strings" +import "git.novox.be/novox/mesh-sdk/go/catalogue" // LauncherSuffix is what a TypeScript entrypoint's launcher is called beside it: index.js is started as // index.serve.mjs (novox/hq ADR 0193), an ES module by its own extension whatever the bundle's package.json says. -const LauncherSuffix = ".serve.mjs" +const LauncherSuffix = catalogue.LauncherSuffix -// Launchers are the launchers a bundle's build writes, by entrypoint, relative to the bundle's root: for TypeScript -// every entrypoint `x.js` is started by `x.serve.mjs` beside it; an entrypoint that is not JavaScript and every other -// language, whose build is already executable, get none. The one rule mesh-build-node, build-agent and the controller -// share (R4e), so the name a build writes is the name a reader expects. +// Launchers are the launchers a bundle's build writes, by entrypoint, relative to the bundle's root +// (catalogue.Launchers): the one rule mesh-build-node, build-agent, the controller and a node taking in a build share +// (R4e, issue 1126), so the name a build writes is the name a reader expects. func Launchers(language string, entrypoints []string) map[string]string { - out := map[string]string{} - if language != "typescript" { - return out - } - for _, entry := range entrypoints { - if strings.HasSuffix(entry, ".js") { - out[entry] = strings.TrimSuffix(entry, ".js") + LauncherSuffix - } - } - return out + return catalogue.Launchers(language, entrypoints) } diff --git a/go/buscompose/buscompose_test.go b/go/buscompose/buscompose_test.go index 6bb593d..c5e6f90 100644 --- a/go/buscompose/buscompose_test.go +++ b/go/buscompose/buscompose_test.go @@ -223,6 +223,31 @@ func TestTheCopiedComposerComposesAsTheControllers(t *testing.T) { if !slices.Contains(perms.Subscribe, VaultSecretChanged) { t.Errorf("node one does not hear the vault's secret-changed: %v", perms.Subscribe) } + // And what a module on another node serves from its settings, that key alone (mailu's smtp domain, issue 1126). + users, err := Users(Records{Nodes: []string{"one", "two"}, Assigned: map[string][]Declared{ + "two": {{Module: "mailer", Served: []string{"domain", "not.a.token"}}}}}) + if err != nil { + t.Fatal(err) + } + get := func(key string) string { + return "$JS.API.DIRECT.GET.KV_" + ResolvedBucket + ".$KV." + ResolvedBucket + "." + key + } + for _, u := range users { + if u.Kind != KindNode { + continue + } + perms, err := PermissionsFor(u) + if err != nil { + t.Fatal(err) + } + if u.Node == "one" && (!reflect.DeepEqual(u.ServedReads, []string{"two.mailer.domain"}) || !MayPublish(perms, get("two.mailer.domain")) || + MayPublish(perms, get("two.mailer.sitename"))) { + t.Errorf("node one reads %v of two's resolved values", u.ServedReads) + } + if u.Node == "two" && len(u.ServedReads) != 0 { + t.Errorf("node two is granted its own served keys apart: %v", u.ServedReads) + } + } }) t.Run("the controller's golden server configuration", func(t *testing.T) { seat := Seat{Name: "telegram-sender", Accepts: []string{"send"}, Emits: []string{"delivered", "failed"}} diff --git a/go/buscompose/nats.go b/go/buscompose/nats.go index b548eea..f65cd70 100644 --- a/go/buscompose/nats.go +++ b/go/buscompose/nats.go @@ -217,6 +217,10 @@ type Principal struct { // ADR 0547 §2, ADR 0550 §5, issue 1126): its modules' own secrets sealed to its key, for the compositions it makes // itself. Set while the vault's holder serves it. AsksItsOwnSecrets bool + // ServedReads are, for a machine principal, the keys of mesh-config's resolved values that other nodes' modules serve + // from (`..`, catalogue.ServedSettings): what every consumer is told, and so what a node composing + // its own modules reads of a provider elsewhere (issue 1126), each key alone. Sorted, none twice. + ServedReads []string // FollowsGrantsUserList is the machine principal of the node holding mesh-broker: its node-engine reads the grants // part's sealed user list, one key and an object by name (novox/hq ADR 0547 clause 7, GrantsUserListFollows). FollowsGrantsUserList bool @@ -898,6 +902,12 @@ func PermissionsFor(p Principal) (Permissions, error) { // manifests, the builder's builds (the manifest of a build its want names), and mesh-config's resolved values of // its own node alone. Read only. pub = append(pub, NodeComposeReads(p.Node)...) + // And, of the other nodes' resolved values, the keys their modules serve from (issue 1126), each one key. + served, err := ServedReadGrants(p.ServedReads) + if err != nil { + return Permissions{}, err + } + pub = append(pub, served...) // And an object of the declaration bucket by its name, and nothing of the bucket's list or writing // (novox/hq ADR 0486 §2, issue 1697). fileReads, err := DeclarationFilesReads(p.Node) diff --git a/go/buscompose/state.go b/go/buscompose/state.go index 3d1171a..d795fda 100644 --- a/go/buscompose/state.go +++ b/go/buscompose/state.go @@ -325,3 +325,29 @@ func NodeComposeReads(node string) []string { } return append(out, readGrants(ResolvedBucket, node+".>")...) } + +// ServedReadGrants is the read of each key of mesh-config's resolved values named, `..` +// (Principal.ServedReads, issue 1126): a get and a watch of that key alone. Each part is checked as every subject token +// is, so a key cannot widen the grant. +func ServedReadGrants(keys []string) ([]string, error) { + var out []string + seen := map[string]bool{} + for _, k := range keys { + parts := strings.Split(k, ".") + if len(parts) != 3 { + return nil, fmt.Errorf("%q is not ..", k) + } + for _, part := range parts { + if !safeSubject.MatchString(part) { + return nil, fmt.Errorf("%q cannot be part of a subject: a permission is a subject pattern, and this would widen it", part) + } + } + for _, g := range readGrants(ResolvedBucket, k) { + if !seen[g] { + seen[g] = true + out = append(out, g) + } + } + } + return out, nil +} diff --git a/go/buscompose/users.go b/go/buscompose/users.go index 56dbcc9..71be224 100644 --- a/go/buscompose/users.go +++ b/go/buscompose/users.go @@ -5,6 +5,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/announce" "slices" "sort" + "strings" ) // Every user the composed file should contain, derived from what the mesh knows. @@ -25,6 +26,9 @@ type Declared struct { Emits []string Consumes []string Serves []string + // Served are the config keys its served values and zone name fill from its settings (catalogue.ServedSettings): what + // every node may read of its resolved values where it runs (Principal.ServedReads, issue 1126). + Served []string // Holds are the seats this module claims, with the protocol each seat declares. A seat the // mesh defines for itself declares no protocol, so holding one grants nothing on the bus — // which is right: those seats are about who does a job, not about who may say what. @@ -144,6 +148,7 @@ func Users(r Records) ([]Principal, error) { out := []Principal{{Kind: KindController, Uses: asksSeatOf(r), AsksIssued: issuedOf(r), AsksTheVault: vaultVerbsOf(r), AsksTheConfig: servedOf(r, VerbsTheControllerAsksTheConfig), FilesFor: sortedCopy(r.Nodes)}} + served := servedReadsOf(r) for _, node := range sortedCopy(r.Nodes) { witness := false var checks []string @@ -160,7 +165,7 @@ func Users(r Records) ([]Principal, error) { out = append(out, Principal{Kind: KindNode, Node: node, WitnessesController: witness, Checks: checks, MovesIntoTheVault: servesMoveIn(r), StatesItsKeyToTheVault: servesVerb(r, VaultNodeKey), AsksItsOwnSecrets: servesVerb(r, VaultOwn), - FollowsGrantsUserList: bus}) + FollowsGrantsUserList: bus, ServedReads: servedOn(served, node)}) // **And the machine's provisioner client, on an account of its own** (novox/hq ADR 0349 clause 11): one // per machine that runs it, delivered to the module running it. for _, d := range r.Assigned[node] { @@ -459,3 +464,32 @@ func declaredStreamsOf(r Records) []string { sort.Strings(out) return out } + +// servedReadsOf is every key of mesh-config's resolved values a module assigned on a node serves from, +// `..`, sorted: each part a subject token, so a key that cannot be one is granted nothing (and the +// node composing a consumer of it refuses, naming it), never a wider read. +func servedReadsOf(r Records) []string { + var out []string + for node, declared := range r.Assigned { + for _, d := range declared { + for _, k := range d.Served { + if safeSubject.MatchString(node) && safeSubject.MatchString(d.Module) && safeSubject.MatchString(k) { + out = append(out, node+"."+d.Module+"."+k) + } + } + } + } + sort.Strings(out) + return slices.Compact(out) +} + +// servedOn is, of the served keys, those of other nodes than node: its own slice it reads whole. +func servedOn(served []string, node string) []string { + var out []string + for _, k := range served { + if !strings.HasPrefix(k, node+".") { + out = append(out, k) + } + } + return out +} diff --git a/go/catalogue/launchers.go b/go/catalogue/launchers.go new file mode 100644 index 0000000..9e9e5b3 --- /dev/null +++ b/go/catalogue/launchers.go @@ -0,0 +1,25 @@ +package catalogue + +import "strings" + +// LauncherSuffix is what a TypeScript entrypoint's launcher is called beside it: index.js is started as +// index.serve.mjs (novox/hq ADR 0193), an ES module by its own extension whatever the bundle's package.json says. +const LauncherSuffix = ".serve.mjs" + +// Launchers are the launchers a bundle's build writes, by entrypoint, relative to the bundle's root: for TypeScript +// every entrypoint `x.js` is started by `x.serve.mjs` beside it; an entrypoint that is not JavaScript and every other +// language, whose build is already executable, get none. The one rule mesh-build-node, build-agent, the controller and +// a node taking in a build share (R4e, issue 1126; buildstep.Launchers says it for the build modules), so the name a +// build writes is the name a reader expects. +func Launchers(language string, entrypoints []string) map[string]string { + out := map[string]string{} + if language != "typescript" { + return out + } + for _, entry := range entrypoints { + if strings.HasSuffix(entry, ".js") { + out[entry] = strings.TrimSuffix(entry, ".js") + LauncherSuffix + } + } + return out +} diff --git a/go/catalogue/settings.go b/go/catalogue/settings.go index 9d86627..03efee3 100644 --- a/go/catalogue/settings.go +++ b/go/catalogue/settings.go @@ -132,6 +132,30 @@ func Settle(base map[string]any, layers []Layer) (map[string]any, error) { return filledFromSettings(deepCopy(base), layers, "what is served") } +// ServedSettings is every key a module's served values and its zone's name fill from its settings (`${setting:}` +// in a top-level string, as Settle fills them), sorted: what every consumer of the module is told, wherever it runs, +// and so the one part of a module's config values on its node that a node composing a consumer elsewhere reads (issue +// 1126). None for a module that serves only literals. +func ServedSettings(m Manifest) []string { + seen := map[string]bool{} + note := func(v any) { + if s, ok := v.(string); ok { + for _, k := range settingsUsed(s) { + seen[k] = true + } + } + } + for _, values := range m.Serves { + for _, v := range values { + note(v) + } + } + if m.Zone != nil { + note(m.Zone.Name) + } + return sortedKeys(seen) +} + // overridden lays settings over a contribution: a map whose keys are its contract. Only the keys the // map already declares are touched; the rest of a layer is somebody else's business (a file's, // another destination's) and is left to reach it there. diff --git a/go/compose/frombus/frombus.go b/go/compose/frombus/frombus.go index 07f1236..0c1c28f 100644 --- a/go/compose/frombus/frombus.go +++ b/go/compose/frombus/frombus.go @@ -14,10 +14,13 @@ // - what each node observes of itself (its tunnel key, capabilities, keys, certificate, links and networks): the // node's own have key, `.node` (HaveNode.Record); the tunnel found at a node's adoption and the names of the hub's peers are decided, // in its placement, and whether the hub took the tunnel over and who enrolled under a peer's key are read from both; -// - every module's manifest: the indexer's `modules.` and its manifests by digest, or the digest the -// composing node's want names where it runs another build (Scope.Kept); +// - every module's manifest: the indexer's `modules.` and its manifests by digest; for each module placed on +// the composing node, its built manifest instead: the builder's done key at the trunk's FP2, or at the fingerprint +// the node's want names where it runs another build (Scope.KeptAt), resolved by the controller's own take-in rule +// (takein.Manifest), a moved module's templates archive read in the artifact store (issue 1126); // - each module's config values on a node: mesh-config's resolved state, the composing node's slice alone unless the -// caller composes every node (ADR 0547 §2, the prepare step's L4). +// caller composes every node (ADR 0547 §2, the prepare step's L4); of another node, only the keys its modules' +// served values fill (catalogue.ServedSettings), which every consumer is told (issue 1126). // // **Own secrets and pair credentials** come from the vault sealed to the composing node's key (OwnSecrets, Credentials), // for that node alone: as a consumer and as a provider, exactly the keys compose.CredentialsNeeded names. **Not yet @@ -32,6 +35,7 @@ import ( "encoding/json" "fmt" "net/netip" + "slices" "sort" "strings" @@ -40,6 +44,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/declaration" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/overlay" + "git.novox.be/novox/mesh-sdk/go/takein" ) // Reader is the bus as a caller reads it with its own credential. Each method reads one part's state; an error is a @@ -64,6 +69,18 @@ type Reader interface { // and each it grants as a provider, with the copy sealed to it (the vault's `own` with the keys, ADR 0550 §5). A key // the vault holds no value for is left out; the composition then stands in for it, and Compose refuses it. Credentials(ctx context.Context, node string, asConsumer, asProvider []compose.CredentialKey) ([]compose.Credential, []compose.Issued, error) + // Build is a module's build at a fingerprint, the builder's key (delivery.BuilderBuildsBucket, BuilderBuildKey); nil + // when the builder holds none. The composing node composes each of its modules from its build (issue 1126). + Build(ctx context.Context, module, fp string) (*delivery.BuilderBuild, error) + // Archive is the archive at url, a kept reference routed to the artifact store as the composing node reaches it + // (catalogue.Routed), checked against digest: a moved module's templates archive, read back for its texts as the + // controller's take-in reads it (takein.Built). + Archive(ctx context.Context, url, digest string) ([]byte, error) + // Served is, of one module's config values on another node than the composing one (mesh-config's state `resolved`, + // keys `..`), the keys named, decoded: the keys its served values and zone name fill + // (catalogue.ServedSettings), which every consumer is told and so every node may read (issue 1126). Nil when none + // is set. + Served(ctx context.Context, node, module string, keys []string) (map[string]any, error) } // Mesh is what is true of the whole mesh and is no part's state on the bus yet: the private network's range and the @@ -79,8 +96,16 @@ type Mesh struct { } // Scope says who composes. Node is the node composing; only its config values are read unless Every is set, as the -// composition check sets it to compose every node. Kept is, by module, the manifest digest the composing node's wants -// name where it differs from the indexer's trunk: that node composes the module as it runs it. +// composition check sets it to compose every node (of another node's modules, only the values their served values fill +// are read, Reader.Served). KeptAt is, by module, the fingerprint the composing node's want names where it differs from +// the indexer's trunk: that node composes the module from its build at that fingerprint, as it runs it. +// +// **Composed from its build** (issue 1126): each module placed on the composing node, and each in KeptAt, composes from +// its built manifest (takein.Manifest over the builder's done key at the trunk's FP2 or the kept fingerprint), as the +// controller composes from the manifest its take-in resolved: so a moved module is handed to the provisioner, a resource +// naming an artifact names what was built, and a templated file has its text. A placed module the builder holds no done +// build of at the trunk's FP2 composes from the indexer's manifest, as before. Every leaves the catalogue as the indexer +// read it: the check of every node judges placement, and reads no archive. // // Placements, when set, stands in for the indexer's placement of the nodes it names: a node's candidate placement // (delivery.PlacementCandidate), so a node composes in shadow with a catalogue pull request's placement and compares @@ -91,7 +116,7 @@ type Mesh struct { type Scope struct { Node string Every bool - Kept map[string]string + KeptAt map[string]string Placements map[string]*delivery.Placement InOwn []string } @@ -178,15 +203,18 @@ func Input(ctx context.Context, r Reader, mesh Mesh, s Scope) (compose.Input, er } } } + parse := func(raw []byte) (catalogue.Manifest, error) { + if mesh.Predefined != nil { + return catalogue.ParseManifestWith(raw, mesh.Predefined) + } + return catalogue.ParseManifestForInputs(raw) + } read := func(module, digest string) (catalogue.Manifest, error) { raw, err := r.Manifest(ctx, digest) if err != nil { return catalogue.Manifest{}, fmt.Errorf("frombus: the manifest of %s (digest %s): %w", module, digest, err) } - if mesh.Predefined != nil { - return catalogue.ParseManifestWith(raw, mesh.Predefined) - } - return catalogue.ParseManifestForInputs(raw) + return parse(raw) } for _, name := range sortedKeys(modules) { m := modules[name] @@ -230,6 +258,13 @@ func Input(ctx context.Context, r Reader, mesh Mesh, s Scope) (compose.Input, er if values, err = r.Values(ctx, node, m.Module); err != nil { return compose.Input{}, fmt.Errorf("frombus: %s's config values on %s: %w", m.Module, node, err) } + } else if keys := catalogue.ServedSettings(in.Catalogue[m.Module]); len(keys) > 0 { + // What a provider elsewhere serves from its settings is told to every consumer: the composing node reads + // those keys of it, and no other (issue 1126: mailu's smtp domain, a consumer's on shanks). + if values, err = r.Served(ctx, node, m.Module, keys); err != nil { + return compose.Input{}, fmt.Errorf("frombus: what %s serves on %s (%s): %w", m.Module, node, + strings.Join(keys, ", "), err) + } } if layers := m.Layers(node, values); layers != nil { if n.Settings == nil { @@ -245,16 +280,6 @@ func Input(ctx context.Context, r Reader, mesh Mesh, s Scope) (compose.Input, er } n.InOwn[module] = true } - for _, module := range sortedKeys(s.Kept) { - man, err := read(module, s.Kept[module]) - if err != nil { - return compose.Input{}, err - } - if n.Kept == nil { - n.Kept = map[string]catalogue.Manifest{} - } - n.Kept[module] = man - } } in.Nodes = append(in.Nodes, n) in.Holdings = append(in.Holdings, p.Held()...) @@ -284,6 +309,9 @@ func Input(ctx context.Context, r Reader, mesh Mesh, s Scope) (compose.Input, er if _, ok := records[s.Node]; !ok { return compose.Input{}, fmt.Errorf("frombus: %s has written no record of itself yet", s.Node) } + if err := fromBuilds(ctx, r, &in, s, placements[s.Node], modules, read, parse); err != nil { + return compose.Input{}, err + } // Its own modules' secrets, sealed to its own key by the vault: a composition carries them sealed (ADR 0550 §5). // Asked only when a module placed on it declares one: a node none of whose modules has a secret composes // whether or not the vault answers. @@ -325,6 +353,99 @@ func Compose(ctx context.Context, r Reader, mesh Mesh, s Scope, fps map[string]s return compose.CompositionsApplied(in, s.Node, fps) } +// fromBuilds puts in the built manifest of each module placed on the composing node, at the trunk's FP2, and of each +// module it keeps at another build (Scope.KeptAt, into its node's Kept), read from the builder's done key and resolved +// by the one take-in rule (takein.Manifest), a templates archive read in the artifact store as the node reaches it +// (issue 1126). +func fromBuilds(ctx context.Context, r Reader, in *compose.Input, s Scope, p delivery.Placement, modules map[string]delivery.Module, + read func(module, digest string) (catalogue.Manifest, error), parse func([]byte) (catalogue.Manifest, error)) error { + address, addressErr := "", error(nil) + addressed := false + archive := func(ctx context.Context, reference, digest string) ([]byte, error) { + if !addressed { + address, addressErr = compose.ArtifactStoreAddress(*in, s.Node) + addressed = true + } + switch { + case addressErr != nil: + return nil, addressErr + case address == "": + return nil, fmt.Errorf("%s reaches no artifact store", s.Node) + } + return r.Archive(ctx, catalogue.Routed(reference, address), digest) + } + built := func(module, fp string, b *delivery.BuilderBuild) (catalogue.Manifest, error) { + if b.Manifest == "" { + return catalogue.Manifest{}, fmt.Errorf("frombus: the build of %s at %s names no manifest", module, fp) + } + src, err := read(module, b.Manifest) + if err != nil { + return catalogue.Manifest{}, err + } + resolved, err := takein.Manifest(ctx, src, *b, archive) + if err != nil { + return catalogue.Manifest{}, fmt.Errorf("frombus: %s at %s: %w", module, fp, err) + } + // Read back as the mesh holds a built manifest (the controller keeps the resolved manifest's bytes and reads them + // as any manifest): a template's text is then in the manifest's own words, its ${config:…} in the settings form. + raw, err := json.Marshal(resolved) + if err != nil { + return catalogue.Manifest{}, err + } + if resolved, err = parse(raw); err != nil { + return catalogue.Manifest{}, fmt.Errorf("frombus: the built manifest of %s at %s: %w", module, fp, err) + } + return resolved, nil + } + for _, pm := range p.Modules { + m, ok := modules[pm.Module] + if !ok || m.FP2 == "" || s.KeptAt[pm.Module] != "" || slices.Contains(s.InOwn, pm.Module) { + // A module it runs inside its own composition is composed there as before: no build of the new form is what + // it runs (Scope.InOwn). + continue + } + if man, ok := in.Catalogue[pm.Module]; !ok || (man.Build == nil && !man.Templated()) { + continue // nothing is built of it: its manifest is what it composes from + } + b, err := r.Build(ctx, pm.Module, m.FP2) + if err != nil { + return fmt.Errorf("frombus: the build of %s at %s: %w", pm.Module, m.FP2, err) + } + if b == nil || b.State != delivery.BuildDone { + continue // no done build at the trunk's: composed from the indexer's manifest, as before + } + resolved, err := built(pm.Module, m.FP2, b) + if err != nil { + return err + } + in.Catalogue[pm.Module] = resolved + } + for i := range in.Nodes { + if in.Nodes[i].Name != s.Node { + continue + } + for _, module := range sortedKeys(s.KeptAt) { + fp := s.KeptAt[module] + b, err := r.Build(ctx, module, fp) + if err != nil { + return fmt.Errorf("frombus: the build of %s at %s: %w", module, fp, err) + } + if b == nil || b.State != delivery.BuildDone { + return fmt.Errorf("frombus: %s is wanted at %s, of which the builder holds no done build", module, fp) + } + resolved, err := built(module, fp, b) + if err != nil { + return err + } + if in.Nodes[i].Kept == nil { + in.Nodes[i].Kept = map[string]catalogue.Manifest{} + } + in.Nodes[i].Kept[module] = resolved + } + } + return nil +} + // declaresOwnSecrets says a module assigned to node declares an own secret, in the manifest the node composes it from. func declaresOwnSecrets(in compose.Input, node string) bool { for _, n := range in.Nodes { diff --git a/go/compose/frombus/frombus_test.go b/go/compose/frombus/frombus_test.go index ce2623b..20a5ae5 100644 --- a/go/compose/frombus/frombus_test.go +++ b/go/compose/frombus/frombus_test.go @@ -1,6 +1,9 @@ package frombus import ( + "archive/tar" + "bytes" + "compress/gzip" "context" "crypto/ecdh" "crypto/rand" @@ -10,6 +13,7 @@ import ( "fmt" "os" "reflect" + "slices" "strings" "testing" @@ -32,6 +36,39 @@ type bus struct { // asked is every consumer credential asked; the fake refuses the whole ask, as the vault's own does, for a consuming // module its placement does not put on the node. asked []compose.CredentialKey + // builds are the builder's keys, by " "; archives the artifact store's blobs, by digest, and fetched + // every URL asked; served another node's values by ".", and servedRead every ask with its keys. + builds map[string]*delivery.BuilderBuild + archives map[string][]byte + fetched []string + served map[string]map[string]any + servedRead []string + buildsRead []string +} + +func (b *bus) Build(_ context.Context, module, fp string) (*delivery.BuilderBuild, error) { + b.buildsRead = append(b.buildsRead, module) + return b.builds[module+" "+fp], nil +} + +func (b *bus) Archive(_ context.Context, url, digest string) ([]byte, error) { + b.fetched = append(b.fetched, url) + body, ok := b.archives[digest] + if !ok { + return nil, errors.New("the store holds no such blob") + } + return body, nil +} + +func (b *bus) Served(_ context.Context, node, module string, keys []string) (map[string]any, error) { + b.servedRead = append(b.servedRead, node+"."+module+":"+strings.Join(keys, ",")) + out := map[string]any{} + for _, k := range keys { + if v, ok := b.served[node+"."+module][k]; ok { + out[k] = v + } + } + return out, nil } func (b *bus) Placements(context.Context) (map[string]delivery.Placement, error) { @@ -442,11 +479,193 @@ func TestInput(t *testing.T) { t.Error("laptop's media, which declares an own secret, composed without the vault") } + // A module the node's want keeps at another build than the trunk's composes from that build's manifest, resolved with + // what it made (issue 1126), on the composing node alone. kept := mk() - kept.manifests["old"] = xterm - in, err := Input(ctx, kept, Mesh{}, Scope{Node: "laptop", Kept: map[string]string{"xterm": "old"}}) - if err != nil || !reflect.DeepEqual(in.Nodes[1].Kept, map[string]catalogue.Manifest{"xterm": parsed}) || in.Nodes[0].Kept != nil { - t.Errorf("the want's manifest is kept on the composing node alone: %v, %v", in.Nodes, err) + oldXterm := []byte(`{"module":"xterm","version":"1","resources":[{"id":"tools","type":"process","name":"xterm-tools",` + + `"artifact":"tools","run":["./xterm-tools"]}],"build":{"artifacts":[{"name":"tools","kind":"bundle","language":"go","system":"arch","from":"cmd/xterm-tools","binary":"xterm-tools"}]}}`) + kept.manifests["old"] = oldXterm + oldFP := "src2:" + strings.Repeat("9", 64) + toolsRef := "artifact-store://xterm/tools/blobs/sha256:" + strings.Repeat("1", 64) + kept.builds = map[string]*delivery.BuilderBuild{"xterm " + oldFP: {State: delivery.BuildDone, Manifest: "old", + Artifacts: []delivery.BuiltArtifact{{Name: "tools", Kind: "bundle", Reference: toolsRef, Digest: "sha256:" + strings.Repeat("1", 64)}}}} + parsedOld, err := catalogue.ParseManifestForInputs(oldXterm) + if err != nil { + t.Fatal(err) + } + keptBuilt, err := parsedOld.Resolve([]catalogue.Built{{Name: "tools", Kind: "bundle", Reference: toolsRef, Digest: "sha256:" + strings.Repeat("1", 64)}}) + if err != nil { + t.Fatal(err) + } + in, err := Input(ctx, kept, Mesh{}, Scope{Node: "laptop", KeptAt: map[string]string{"xterm": oldFP}}) + if err != nil || !reflect.DeepEqual(in.Nodes[1].Kept, map[string]catalogue.Manifest{"xterm": keptBuilt}) || in.Nodes[0].Kept != nil { + t.Errorf("the want's build is kept on the composing node alone: %v, %v", in.Nodes, err) + } + + // Issue 1126, the seed's refusals. restic and the provisioner's client as the catalogue holds them (mesh-catalog + // c05aa5e6), restic moved with an own secret the vault leaves out (it hands it to the client, mesh-vault ownnode.go + // handed): composed from the indexer's manifest restic is not handed and stands in for that secret, as novox's, + // ace's, g14's and shanks's answers refused; composed from its build, it is handed, its templates archive read + // where the node reaches the artifact store, and the composition is the one the controller's take-in gives. + resticRaw, err := os.ReadFile("testdata/catalogue-c05aa5e6-restic.module.json") + if err != nil { + t.Fatal(err) + } + clientRaw, err := os.ReadFile("testdata/catalogue-c05aa5e6-mesh-provisioner-client.module.json") + if err != nil { + t.Fatal(err) + } + store := []byte(`{"module":"store","version":"1","provides":[{"name":"artifact-store","scope":"mesh"}],` + + `"serves":{"artifact-store":{"port":5000}}}`) + texts := map[string]string{} + var packed bytes.Buffer + zw := gzip.NewWriter(&packed) + tw := tar.NewWriter(zw) + for _, f := range []string{"backups.conf", "data.conf", "retention.conf"} { + text, err := os.ReadFile("testdata/catalogue-c05aa5e6-restic.files/" + f) + if err != nil { + t.Fatal(err) + } + texts[f] = string(text) + if err := tw.WriteHeader(&tar.Header{Name: "./" + f, Mode: 0o644, Size: int64(len(text)), Typeflag: tar.TypeReg}); err != nil { + t.Fatal(err) + } + if _, err := tw.Write(text); err != nil { + t.Fatal(err) + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if err := zw.Close(); err != nil { + t.Fatal(err) + } + archiveDigest := delivery.ManifestDigest(packed.Bytes()) + digestOf := func(c string) string { return "sha256:" + strings.Repeat(c, 64) } + movedFps := map[string]string{"restic": "src2:" + strings.Repeat("3", 64), "mesh-provisioner-client": "src2:" + strings.Repeat("4", 64), + "store": "src2:" + strings.Repeat("5", 64)} + resticArtifacts := []delivery.BuiltArtifact{{Name: "tools", Kind: "bundle", Reference: "artifact-store://restic/tools/blobs/" + digestOf("6"), Digest: digestOf("6")}, + {Name: "files", Kind: "archive", Reference: "artifact-store://restic/files/blobs/" + archiveDigest, Digest: archiveDigest}} + clientArtifacts := []delivery.BuiltArtifact{{Name: "client", Kind: "bundle", Reference: "artifact-store://mesh-provisioner-client/client/blobs/" + digestOf("7"), Digest: digestOf("7")}} + movedMesh := func(built bool) *bus { + b := one() + p := b.placements["n1"] + p.Modules = []delivery.PlacedModule{{Module: "store"}, {Module: "mesh-provisioner-client"}, {Module: "restic"}} + p.Decided.Account, p.Decided.AccountHome = "operator", "/home/operator" + b.placements["n1"] = p + rec := b.records["n1"] + rec.Contract, rec.Capabilities = compose.HandedContract, []string{"systemd", "package-manager"} + b.records["n1"] = rec + b.modules = map[string]delivery.Module{} + b.manifests = map[string][]byte{} + for module, raw := range map[string][]byte{"restic": resticRaw, "mesh-provisioner-client": clientRaw, "store": store} { + b.modules[module] = delivery.Module{FP2: movedFps[module], Manifest: delivery.ManifestDigest(raw)} + b.manifests[delivery.ManifestDigest(raw)] = raw + } + // The vault's own answer: the client's own secret; restic's is the client's to ask, so own leaves it out. + b.own = []compose.OwnSecretRow{{Node: "n1", Module: "mesh-provisioner-client", Name: "provisioner-broker", Origin: "made", SealedTo: "sk", Sealed: `"box"`}} + if built { + b.builds = map[string]*delivery.BuilderBuild{ + "restic " + movedFps["restic"]: {State: delivery.BuildDone, Manifest: delivery.ManifestDigest(resticRaw), Artifacts: resticArtifacts}, + "mesh-provisioner-client " + movedFps["mesh-provisioner-client"]: {State: delivery.BuildDone, Manifest: delivery.ManifestDigest(clientRaw), Artifacts: clientArtifacts}, + "store " + movedFps["store"]: {State: delivery.BuildDone, Manifest: delivery.ManifestDigest(store)}, + } + b.archives = map[string][]byte{archiveDigest: packed.Bytes()} + } + return b + } + if _, _, err := Compose(ctx, movedMesh(false), Mesh{}, Scope{Node: "n1"}, movedFps); err == nil || !strings.Contains(err.Error(), `restic needs a secret called "repository"`) { + t.Errorf("restic composed from the indexer's manifest: %v, want its own secret refused", err) + } + builtBus := movedMesh(true) + fromBuild, _, err := Compose(ctx, builtBus, Mesh{}, Scope{Node: "n1"}, movedFps) + if err != nil { + t.Fatalf("restic composed from its build: %v", err) + } + if !reflect.DeepEqual(builtBus.fetched, []string{"http://127.0.0.1:5000/v2/restic/files/blobs/" + archiveDigest}) { + t.Errorf("the templates archive was read at %v", builtBus.fetched) + } + resticKey, _ := delivery.ComposedKey("n1", "restic", movedFps["restic"]) + if c := fromBuild[resticKey]; len(c.Filled) != 4 { // its three templated files and its own secret + t.Errorf("restic is not handed to the provisioner: its filled files are %v", c.Filled) + } + // A done key an artifact of which names no reference is waited for, in the controller's words (ADR 0498 §8). + unreferenced := movedMesh(true) + unreferenced.builds["restic "+movedFps["restic"]].Artifacts = []delivery.BuiltArtifact{{Name: "tools", Kind: "bundle", Digest: digestOf("6")}, resticArtifacts[1]} + if _, _, err := Compose(ctx, unreferenced, Mesh{}, Scope{Node: "n1"}, movedFps); err == nil || !strings.Contains(err.Error(), "the build of restic is done and waits for the reference of tools") { + t.Errorf("a done key with an unreferenced artifact: %v", err) + } + // A module the node runs inside its own composition (the fold) composes from the indexer's manifest though a done + // build of it is at the trunk's FP2: its build is not read. + folding := movedMesh(true) + foldIn, err := Input(ctx, folding, Mesh{}, Scope{Node: "n1", InOwn: []string{"mesh-provisioner-client"}}) + if err != nil { + t.Fatal(err) + } + clientParsed, err := catalogue.ParseManifestForInputs(clientRaw) + if err != nil { + t.Fatal(err) + } + if slices.Contains(folding.buildsRead, "mesh-provisioner-client") || !reflect.DeepEqual(foldIn.Catalogue["mesh-provisioner-client"], clientParsed) { + t.Errorf("a folded module was taken from its build: builds read %v", folding.buildsRead) + } + // The controller's: its take-in resolves each manifest with the key's artifacts and the archive's texts, and composes + // from those over the same input. + storeIn, err := Input(ctx, movedMesh(false), Mesh{}, Scope{Node: "n1"}) + if err != nil { + t.Fatal(err) + } + for module, built := range map[string][]catalogue.Built{ + "restic": {{Name: "tools", Kind: "bundle", Reference: resticArtifacts[0].Reference, Digest: digestOf("6")}, + {Name: "files", Kind: "archive", Reference: resticArtifacts[1].Reference, Digest: archiveDigest, Texts: texts}}, + "mesh-provisioner-client": {{Name: "client", Kind: "bundle", Reference: clientArtifacts[0].Reference, Digest: digestOf("7")}}, + } { + resolved, err := storeIn.Catalogue[module].Resolve(built) + if err != nil { + t.Fatal(err) + } + // Kept as its bytes and read again, as the controller's store holds a build's manifest. + raw, _ := json.Marshal(resolved) + if storeIn.Catalogue[module], err = catalogue.ParseManifestForInputs(raw); err != nil { + t.Fatal(err) + } + } + storeIn.OwnSecrets = movedMesh(true).own + fromStoreMoved, _, err := compose.Compositions(storeIn, "n1", movedFps, compose.Reading) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(fromBuild, fromStoreMoved) { + gj, _ := json.MarshalIndent(fromBuild, "", " ") + wj, _ := json.MarshalIndent(fromStoreMoved, "", " ") + t.Errorf("composed from the build differs from the controller's take-in:\n got %s\nwant %s", gj, wj) + } + + // And what a provider on another node serves from its settings (mailu's smtp domain, issue 1126: shanks refused + // '${setting:domain} ... nothing sets "domain"'): read for that provider's served keys alone, through Served. + mailer := []byte(`{"module":"mailer","version":"1","config":{"domain":{"type":"string","description":"the mail domain","required":true}},` + + `"provides":[{"name":"smtp","scope":"mesh"}],"serves":{"smtp":{"port":587,"domain":"${config:domain}"}}}`) + withMailer := func(set bool) *bus { + b := mk() + p := b.placements["anchor"] + p.Modules = append(p.Modules, delivery.PlacedModule{Module: "mailer"}) + b.placements["anchor"] = p + b.modules["mailer"] = delivery.Module{Manifest: delivery.ManifestDigest(mailer)} + b.manifests[delivery.ManifestDigest(mailer)] = mailer + if set { + b.served = map[string]map[string]any{"anchor.mailer": {"domain": "example.org"}} + } + return b + } + if _, _, err := Compose(ctx, withMailer(false), Mesh{}, Scope{Node: "laptop"}, fps); err == nil || !strings.Contains(err.Error(), `nothing sets "domain"`) { + t.Errorf("a provider's served setting nobody sets: %v", err) + } + served := withMailer(true) + if _, _, err := Compose(ctx, served, Mesh{}, Scope{Node: "laptop"}, fps); err != nil { + t.Errorf("laptop with what anchor's mailer serves: %v", err) + } + if !reflect.DeepEqual(served.servedRead, []string{"anchor.mailer:domain"}) || slices.Contains(served.valuesRead, "anchor.mailer") { + t.Errorf("laptop read %v of other nodes (and %v of config values): the served keys of anchor's mailer alone", served.servedRead, served.valuesRead) } for name, c := range map[string]struct { @@ -454,10 +673,10 @@ func TestInput(t *testing.T) { scope Scope says string }{ - "no record": {func(b *bus) { delete(b.records, "laptop") }, Scope{Node: "anchor"}, "laptop, whose agent has written no record"}, - "no placement": {func(b *bus) {}, Scope{Node: "stranger"}, "stranger has no placement"}, - "a manifest unread": {func(b *bus) { delete(b.manifests, digest) }, Scope{Node: "laptop"}, "the manifest of xterm"}, - "a kept one unread": {func(b *bus) {}, Scope{Node: "laptop", Kept: map[string]string{"xterm": "gone"}}, "digest gone"}, + "no record": {func(b *bus) { delete(b.records, "laptop") }, Scope{Node: "anchor"}, "laptop, whose agent has written no record"}, + "no placement": {func(b *bus) {}, Scope{Node: "stranger"}, "stranger has no placement"}, + "a manifest unread": {func(b *bus) { delete(b.manifests, digest) }, Scope{Node: "laptop"}, "the manifest of xterm"}, + "a kept one unbuilt": {func(b *bus) {}, Scope{Node: "laptop", KeptAt: map[string]string{"xterm": "src2:gone"}}, "holds no done build"}, } { b := mk() c.change(b) diff --git a/go/compose/frombus/testdata/catalogue-c05aa5e6-mesh-provisioner-client.module.json b/go/compose/frombus/testdata/catalogue-c05aa5e6-mesh-provisioner-client.module.json new file mode 100644 index 0000000..095128a --- /dev/null +++ b/go/compose/frombus/testdata/catalogue-c05aa5e6-mesh-provisioner-client.module.json @@ -0,0 +1,60 @@ +{ + "module": "mesh-provisioner-client", + "version": "1", + "claims": [ + { + "name": "node-provisioner", + "scope": "node" + } + ], + "own-secrets": { + "provisioner-broker": "${dir:state}/broker" + }, + "secrets-owner": "mesh-provisioner-client", + "resources": [ + { + "id": "account", + "type": "user", + "name": "mesh-provisioner-client", + "shell": "/usr/bin/nologin", + "home": "/var/lib/mesh-provisioner-client" + }, + { + "id": "state", + "type": "directory", + "mode": "0700", + "owner": "mesh-provisioner-client", + "place": ".", + "unknown": "program" + }, + { + "id": "client", + "type": "process", + "name": "mesh-provisioner-client", + "artifact": "client", + "run": [ + "./mesh-provisioner-client" + ], + "env": { + "MESH_PROVISIONER_CREDENTIAL": "${dir:state}/broker", + "MESH_PROVISIONER_STATE": "${dir:state}" + }, + "user": "mesh-provisioner-client", + "health": { + "kind": "unit" + } + } + ], + "build": { + "artifacts": [ + { + "name": "client", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-provisioner-client", + "binary": "mesh-provisioner-client" + } + ] + } +} diff --git a/go/compose/frombus/testdata/catalogue-c05aa5e6-restic.files/backups.conf b/go/compose/frombus/testdata/catalogue-c05aa5e6-restic.files/backups.conf new file mode 100644 index 0000000..2f23f90 --- /dev/null +++ b/go/compose/frombus/testdata/catalogue-c05aa5e6-restic.files/backups.conf @@ -0,0 +1,2 @@ +# What the modules on this machine back up, composed by the mesh (novox/hq to-be 43). Do not edit. +${backup:lines} \ No newline at end of file diff --git a/go/compose/frombus/testdata/catalogue-c05aa5e6-restic.files/data.conf b/go/compose/frombus/testdata/catalogue-c05aa5e6-restic.files/data.conf new file mode 100644 index 0000000..f19e849 --- /dev/null +++ b/go/compose/frombus/testdata/catalogue-c05aa5e6-restic.files/data.conf @@ -0,0 +1,2 @@ +# The data the modules on this machine declare, composed by the mesh (novox/hq ADR 0233). Do not edit. +${backup:data} \ No newline at end of file diff --git a/go/compose/frombus/testdata/catalogue-c05aa5e6-restic.files/retention.conf b/go/compose/frombus/testdata/catalogue-c05aa5e6-restic.files/retention.conf new file mode 100644 index 0000000..cd7954c --- /dev/null +++ b/go/compose/frombus/testdata/catalogue-c05aa5e6-restic.files/retention.conf @@ -0,0 +1,6 @@ +# How many restore points the night keeps, written by the mesh from this machine's settings keep-daily, +# keep-weekly, keep-monthly (novox/hq issue 416) and keep-hand-taken-days (novox/hq issue 453). Replaced on every push; change the settings, never this file. +daily=${config:keep-daily} +weekly=${config:keep-weekly} +monthly=${config:keep-monthly} +hand=${config:keep-hand-taken-days} diff --git a/go/compose/frombus/testdata/catalogue-c05aa5e6-restic.module.json b/go/compose/frombus/testdata/catalogue-c05aa5e6-restic.module.json new file mode 100644 index 0000000..ea5ec8b --- /dev/null +++ b/go/compose/frombus/testdata/catalogue-c05aa5e6-restic.module.json @@ -0,0 +1,199 @@ +{ + "module": "restic", + "version": "1", + "moved": true, + "claims": [ + { + "name": "node-backup", + "scope": "node", + "serves": [ + "backed-up", + "now", + "restore" + ] + } + ], + "emits": [ + "backed-up" + ], + "tools": [ + "backup_health", + "backup_delete_retired", + "backup_deleted" + ], + "own-secrets": { + "repository": "${dir:state}/repository.secret" + }, + "data": { + "own": [ + { + "id": "repository", + "path": "${dir:repository}", + "class": "rebuildable", + "backup": "none", + "why": "the restore points themselves: a copy of data kept elsewhere on this machine, never the only copy of anything; lost, the history goes and nothing live does" + }, + { + "id": "state", + "path": "${dir:state}", + "class": "cache", + "why": "what the last nights and measurements were; the next night writes it again" + } + ] + }, + "config": { + "keep-daily": { + "type": "number", + "description": "How many nightly restore points are kept, a count from 1 to 9999: two weeks undo a mistake noticed within a fortnight. Read by the night, which thins nothing while it does not read (novox/hq issue 416); lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.", + "minimum": 1, + "maximum": 9999, + "trusted": true, + "default": 14 + }, + "keep-weekly": { + "type": "number", + "description": "How many weekly restore points are kept behind the dailies, a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.", + "minimum": 1, + "maximum": 9999, + "trusted": true, + "default": 8 + }, + "keep-monthly": { + "type": "number", + "description": "How many monthly restore points are kept behind the weeklies, a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.", + "minimum": 1, + "maximum": 9999, + "trusted": true, + "default": 6 + }, + "keep-hand-taken-days": { + "type": "number", + "description": "How many days a restore point taken by hand (node-backup.now, or the mesh's before a module moves) is kept at least, a count from 1 to 9999; it is kept until a later good night of its module exists, and this many days in any case, so the next thinning never takes the one taken against a risky change (novox/hq issue 453). Lowering it lets go of the older ones taken by hand at the next thinning, and that cannot be undone.", + "minimum": 1, + "maximum": 9999, + "trusted": true, + "default": 7 + } + }, + "resources": [ + { + "id": "state", + "type": "directory", + "mode": "0700", + "place": ".", + "unknown": "program" + }, + { + "id": "repository", + "type": "directory", + "mode": "0700", + "unknown": "program" + }, + { + "id": "declared", + "type": "file", + "path": "${dir:state}/backups.conf", + "mode": "0600", + "template": "files/backups.conf" + }, + { + "id": "data-declared", + "type": "file", + "path": "${dir:state}/data.conf", + "mode": "0600", + "template": "files/data.conf" + }, + { + "id": "retention", + "type": "file", + "path": "${dir:state}/retention.conf", + "mode": "0644", + "trusted": true, + "template": "files/retention.conf" + }, + { + "id": "tool", + "type": "package", + "package": "restic" + }, + { + "id": "sqlite", + "type": "package", + "package": "sqlite" + }, + { + "id": "night", + "type": "process", + "name": "restic-night", + "artifact": "tools", + "run": [ + "./restic-backups", + "night" + ], + "user": "${machine:account}", + "env": { + "MESH_BACKUP_DECLARED": "${dir:state}/backups.conf", + "MESH_BACKUP_REPOSITORY": "${dir:repository}", + "MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret", + "MESH_BACKUP_STATE": "${dir:state}", + "MESH_BACKUP_DATA": "${dir:state}/data.conf" + }, + "health": { + "kind": "tool", + "tool": "backup_health", + "interval": "60s", + "timeout": "20s", + "looks": 2, + "grace": "60s" + } + }, + { + "id": "measure", + "type": "process", + "name": "restic-measure", + "artifact": "tools", + "run": [ + "./restic-backups", + "measure" + ], + "user": "${machine:account}", + "env": { + "MESH_BACKUP_DECLARED": "${dir:state}/backups.conf", + "MESH_BACKUP_REPOSITORY": "${dir:repository}", + "MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret", + "MESH_BACKUP_STATE": "${dir:state}", + "MESH_BACKUP_DATA": "${dir:state}/data.conf" + }, + "health": { + "kind": "unit" + } + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/restic-backups", + "binary": "restic-backups", + "loads": [ + "restic-backups" + ], + "env": { + "MESH_BACKUP_DECLARED": "${dir:state}/backups.conf", + "MESH_BACKUP_REPOSITORY": "${dir:repository}", + "MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret", + "MESH_BACKUP_STATE": "${dir:state}", + "MESH_BACKUP_DATA": "${dir:state}/data.conf" + } + }, + { + "name": "files", + "kind": "archive", + "from": "files" + } + ] + } +} diff --git a/go/compose/rendering.go b/go/compose/rendering.go index 468683d..4f0e648 100644 --- a/go/compose/rendering.go +++ b/go/compose/rendering.go @@ -325,6 +325,13 @@ func (c *composer) artifactStoreOnNetwork(on map[string]bool) (node, port string return "", "", false, nil } +// ArtifactStoreAddress is the artifact store as node reaches it, by the rule its composition routes every kept reference +// with (artifactStoreAddress): empty when node reaches none. A node composing its own modules reads a templates archive +// there (issue 1126). +func ArtifactStoreAddress(in Input, node string) (string, error) { + return newComposer(in).artifactStoreAddress(node) +} + // artifactStoreAddress is the artifact store as `forNode` reaches it: `.internal:` over the private network, or // `127.0.0.1:` for the node that holds the store itself when nothing is on the network yet, and empty for any other // (novox/hq 04-ISSUES/102). diff --git a/go/takein/takein.go b/go/takein/takein.go new file mode 100644 index 0000000..657ab45 --- /dev/null +++ b/go/takein/takein.go @@ -0,0 +1,134 @@ +// Package takein is the one rule that turns a builder's done key into the manifest the mesh composes a module from +// (issue 1126): the module's manifest resolved with what its build made. It is mesh-controller's take-in of a done key +// (cmd/mesh-controller/key_takein.go builtOfKey, unpackTexts and Manifest.Resolve) moved into the SDK, so a node agent +// composing its own modules (novox/hq ADR 0547 §2) composes from the same built manifest the controller composes from: +// every artifact a resource names resolved to its reference and digest, a TypeScript bundle's launchers, and, for a +// moved module, each template's text read back from its templates archive and what its build keeps for the +// provisioner (catalogue.Handed). Composing from the manifest the indexer read instead hands no moved module and +// leaves every artifact unbuilt. +// +// It reads nothing itself: the archive is read through the caller's Archive, from wherever the caller reaches the +// artifact store. +package takein + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "errors" + "fmt" + "io" + "strings" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// Archive reads the archive a recorded reference names (catalogue.Recorded: the artifact store's address left out), +// checked against its digest. +type Archive func(ctx context.Context, reference, digest string) ([]byte, error) + +// Manifest is m, as its manifest at b's fingerprint reads, resolved with what b made: the built manifest. b must be done +// and every artifact must name its reference. +// A templated module's templates archive is read through archive; a module without templates reads none, and archive +// may then be nil. +func Manifest(ctx context.Context, m catalogue.Manifest, b delivery.BuilderBuild, archive Archive) (catalogue.Manifest, error) { + if b.State != delivery.BuildDone { + return catalogue.Manifest{}, fmt.Errorf("takein: the build of %s is %s, not done: nothing to resolve it with", m.Module, b.State) + } + // **A done key whose artifact names no reference is waited for, never composed from** (novox/hq ADR 0498 §8), in the + // controller's words (mesh-controller internal/builderkeys Key.Says): a key done before R4e, or a candidate's image or + // package not yet built again for the trunk. + var missing []string + for _, a := range b.Artifacts { + if strings.TrimSpace(a.Reference) == "" { + missing = append(missing, a.Name) + } + } + if len(missing) > 0 { + return catalogue.Manifest{}, fmt.Errorf("takein: the build of %s is done and waits for the reference of %s", + m.Module, strings.Join(missing, ", ")) + } + built, err := Built(ctx, m, b.Artifacts, archive) + if err != nil { + return catalogue.Manifest{}, err + } + resolved, err := m.Resolve(built) + if err != nil { + return catalogue.Manifest{}, fmt.Errorf("takein: the build of %s does not resolve: %w", m.Module, err) + } + return resolved, nil +} + +// Built is each of a done key's artifacts as catalogue.Manifest.Resolve reads it: its recorded reference and digest, a +// TypeScript bundle's launchers by the build modules' one rule (catalogue.Launchers), and, for the archive a module packs +// its templates into, each template's text read back from that archive. +func Built(ctx context.Context, m catalogue.Manifest, artifacts []delivery.BuiltArtifact, archive Archive) ([]catalogue.Built, error) { + declared := map[string]catalogue.Artifact{} + if m.Build != nil { + for _, a := range m.Build.Artifacts { + declared[a.Name] = a + } + } + templates := "" + if m.Templated() { + templates = catalogue.TemplatesArtifact(m) + } + var out []catalogue.Built + for _, a := range artifacts { + d := declared[a.Name] + kind := a.Kind + if kind == "" { + kind = d.Kind + } + b := catalogue.Built{Name: a.Name, Kind: kind, Reference: catalogue.Recorded(a.Reference), Digest: a.Digest} + if launchers := catalogue.Launchers(d.Language, d.Entrypoints); len(launchers) > 0 { + b.Launchers = launchers + } + if a.Name == templates && templates != "" { + if archive == nil { + return nil, fmt.Errorf("takein: %s packs its templates into %q, and nothing here reads the artifact store", + m.Module, a.Name) + } + body, err := archive(ctx, b.Reference, b.Digest) + if err != nil { + return nil, fmt.Errorf("takein: the templates archive %q of %s cannot be read from the artifact store: %w", + a.Name, m.Module, err) + } + if b.Texts, err = UnpackTexts(body); err != nil { + return nil, fmt.Errorf("takein: the templates archive %q of %s cannot be read back: %w", a.Name, m.Module, err) + } + } + out = append(out, b) + } + return out, nil +} + +// UnpackTexts is every regular file of a packed templates archive (gzip over tar), by its path inside it. +func UnpackTexts(body []byte) (map[string]string, error) { + zipped, err := gzip.NewReader(bytes.NewReader(body)) + if err != nil { + return nil, err + } + defer zipped.Close() + reader := tar.NewReader(zipped) + texts := map[string]string{} + for { + header, err := reader.Next() + if errors.Is(err, io.EOF) { + return texts, nil + } + if err != nil { + return nil, err + } + if header.Typeflag != tar.TypeReg { + continue + } + text, err := io.ReadAll(io.LimitReader(reader, 16<<20)) + if err != nil { + return nil, err + } + texts[strings.TrimPrefix(header.Name, "./")] = string(text) + } +}