diff --git a/internal/builder/builder.go b/internal/builder/builder.go index 43ed91f..fdcbb64 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -968,6 +968,26 @@ func compile(ctx context.Context, run Runner, tree string, chain Toolchain, if _, err := run(ctx, tree, "docker", invocation...); err != nil { return "", err } + if chain.Dependencies != "" { + // **What the bundle runs with, from the image it was compiled in** (Toolchain.Dependencies). + // A second run in the same image rather than a shell wrapped around the compiler: the + // compile line stays a plain command a reader can run by hand, and the copy is one more + // plain command beside it. Refused by name when the image carries no such directory — an + // older toolchain image — because a bundle packed without its dependencies starts nowhere + // and says so three layers away from here. + copying := []string{ + "run", "--rm", + "--volume", tree + ":" + within, + "--workdir", within, + base, + "sh", "-c", + `test -d "$1" || { echo "the toolchain image carries no $1: it predates the mesh shipping a bundle's dependencies, rebuild $2 first" >&2; exit 1; }; cp -a "$1/." "$3/"`, + "dependencies", chain.Dependencies, chain.Base, out, + } + if _, err := run(ctx, tree, "docker", copying...); err != nil { + return "", fmt.Errorf("copying the %s dependencies a bundle runs with: %w", chain.Language, err) + } + } return filepath.Join(tree, out), nil } diff --git a/internal/builder/bundle_test.go b/internal/builder/bundle_test.go index da718fd..7872da2 100644 --- a/internal/builder/bundle_test.go +++ b/internal/builder/bundle_test.go @@ -82,6 +82,41 @@ func TestABundleIsCompiledAndPackedWithNoDockerfile(t *testing.T) { if !strings.HasPrefix(digest, "sha256:") { t.Fatalf("the bundle was not pinned: %v", got.Manifest.Resources[0]) } + + // **And what it runs with, from the image it was compiled in** (novox/hq to-be 38 WP3). A + // second run in the same toolchain image copies the toolchain's runtime directory — the + // `"type": "module"` package.json and the pruned node_modules — into the output's root, and + // refuses by name when the image carries none rather than packing a bundle that starts nowhere. + var copied string + for _, line := range r.ran { + if strings.HasPrefix(line, "docker run") && strings.Contains(line, "/app/runtime") { + copied = line + } + } + if copied == "" { + t.Fatalf("the bundle's dependencies were not copied in after the compile:\n%s", strings.Join(r.ran, "\n")) + } + if !strings.Contains(copied, "mesh-tools/build@sha256:") || !strings.Contains(copied, "predates") || + !strings.Contains(copied, Out("code")) { + t.Fatalf("the copy does not run in the same toolchain, refuse an older image by name, or land in the artifact's output: %s", copied) + } + if strings.Index(strings.Join(r.ran, "\n"), "--outDir") > strings.Index(strings.Join(r.ran, "\n"), "/app/runtime") { + t.Fatal("the dependencies were copied before the compile wrote its output") + } +} + +// A language whose bundle carries its own dependencies copies nothing in: a Go binary is static. +func TestOnlyALanguageWithARuntimeDirectoryCopiesDependenciesIn(t *testing.T) { + ts, _ := ToolchainFor("typescript") + if ts.Dependencies != "/app/runtime" { + t.Fatalf("typescript bundles run with %q", ts.Dependencies) + } + for _, language := range []string{"go", "python"} { + chain, _ := ToolchainFor(language) + if chain.Dependencies != "" { + t.Fatalf("%s copies %q into every bundle, and its bundles carry their own", language, chain.Dependencies) + } + } } // **Refused before anything is built, naming what to build first.** A base the mesh has not built @@ -145,9 +180,13 @@ func TestTwoBundlesInOneModuleArePackedSeparately(t *testing.T) { t.Fatalf("a module with two bundles did not build: %v", err) } - // Compiled into two different places. + // Compiled into two different places. Only the compile lines: the copy of each bundle's + // dependencies names the same directory again, deliberately. var outputs []string for _, line := range r.ran { + if !strings.Contains(line, "--outDir") { + continue + } for _, part := range strings.Fields(line) { if strings.HasPrefix(part, ".mesh-build/") { outputs = append(outputs, part) diff --git a/internal/builder/toolchain.go b/internal/builder/toolchain.go index 0e2d50a..5df2d3f 100644 --- a/internal/builder/toolchain.go +++ b/internal/builder/toolchain.go @@ -57,6 +57,23 @@ type Toolchain struct { // carrying its debug info. The mistake was believing a comment rather than reading the file it // produced (novox/hq 04-ISSUES/161). LinkerFlags []string + // Dependencies is a directory inside the toolchain image whose contents a bundle in this + // language runs with, copied whole into the compiled output's root after the compile. + // + // **A bundle that compiles is not yet a bundle that runs.** The compiler resolves `import + // "nats"` from the toolchain image's own node_modules and the pack takes only what the compiler + // wrote, so what a machine unpacked could not find a single dependency — and no TypeScript bundle + // had ever run live to show it (novox/hq to-be 38 WP3). For TypeScript the directory holds a + // `package.json` saying `"type": "module"` — Node reads a bare `.js` as CommonJS otherwise, so a + // bundle with its dependencies and without that line still fails to start — and the pruned, + // production-only node_modules the runtime itself ships with: the SDK's and the runtime's + // dependencies, and nothing module-specific yet (novox/hq ADR 0188 §5: a skeleton; a module's + // own npm dependencies are a later step). Empty for a language whose bundle carries its own — + // a Go binary is static, a Python bundle is installed with its dependencies. + // + // A toolchain image without the directory fails the build by name rather than packing a bundle + // that starts nowhere: the image predates this and must be rebuilt first. + Dependencies string // SystemStamp is the variable this language's linker fills with the artifact's declared system, // for a language whose binaries are pinned to one at link time (novox/hq ADR 0005). // @@ -118,9 +135,10 @@ var toolchains = []Toolchain{ "--module", "NodeNext", "--moduleResolution", "NodeNext", "--target", "ES2022", "--rootDir", ".", }, - OutputFlag: "--outDir", - Unit: UnitSources, - SourceExt: ".ts", + OutputFlag: "--outDir", + Unit: UnitSources, + SourceExt: ".ts", + Dependencies: "/app/runtime", }, { Language: "go", diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index 14333ba..03846bd 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -508,7 +508,17 @@ func (r Resolution) compose(with Rendering, owner map[string]string, return nil, fmt.Errorf( "%s needs a secret called %q and none was made for it", m.Module, name) } - first = append(first, ownedBy(m.SecretsOwner, map[string]any{ + // The runtime's credential belongs to the account the runtime runs as (novox/hq ADR 0175, + // to-be 38 WP3): its process is composed `user: ` where the node has one, and a + // root-owned 0600 file is one that process cannot read. Composed here rather than said in + // the manifest, because a manifest cannot say ${machine:account} safely — a node with no + // account has nothing to resolve it to, and then the runtime runs as root and the file + // stays root's. + owner := m.SecretsOwner + if m.Module == RuntimeModule && r.Account != "" { + owner = r.Account + } + first = append(first, ownedBy(owner, map[string]any{ "id": NeedID(name), "type": "file", "path": m.OwnSecrets[name].Path, "sealed": sealed, })) } diff --git a/internal/catalogue/runtime_test.go b/internal/catalogue/runtime_test.go index 8dc73b6..efac9aa 100644 --- a/internal/catalogue/runtime_test.go +++ b/internal/catalogue/runtime_test.go @@ -160,6 +160,11 @@ func TestTheMachineRunsOneRuntimeLoadingEveryDeliveredBundle(t *testing.T) { if env[RuntimeOperatorAccount] != "ops" || env[RuntimeOperatorHome] != "/home/ops" || process["user"] != "ops" { t.Errorf("the operator is not handed to the runtime: %v as %v", env, process["user"]) } + // The credential the process reads belongs to the account it runs as, or it could not read it + // (to-be 38 WP3); other modules' secrets are left as their manifests say. + if credential := fileNamed(out, RuntimeModule+"."+NeedID("broker")); credential == nil || credential["owner"] != "ops" { + t.Errorf("the runtime's credential is not the account's to read: %v", credential) + } restarts := fmt.Sprint(process["restart-on"]) for _, want := range []string{"nftables." + BundleID("tools"), "showcase." + BundleID("code"), RuntimeModule + "." + NeedID("broker")} { if !strings.Contains(restarts, want) { @@ -185,6 +190,9 @@ func TestTheMachineRunsOneRuntimeLoadingEveryDeliveredBundle(t *testing.T) { if _, set := process["user"]; set { t.Error("a user was set on a machine with no account") } + if credential := fileNamed(out, RuntimeModule+"."+NeedID("broker")); credential == nil || credential["owner"] != nil { + t.Errorf("the runtime's credential was given an owner on a machine with no account: %v", credential) + } }) t.Run("a runtime module built wrong is refused by name", func(t *testing.T) { diff --git a/internal/inventory/catalogue.go b/internal/inventory/catalogue.go index 0c1def5..b6d23c1 100644 --- a/internal/inventory/catalogue.go +++ b/internal/inventory/catalogue.go @@ -66,11 +66,15 @@ func (s Source) Current() bool { // gains a requirement, a claim, a resource. What matters is that the change is visible the next // time a node is resolved, which it is. func (i *Inventory) RegisterModule(ctx context.Context, m catalogue.Manifest, from Source) error { - // **Once the node's tool runtime is in the catalogue, the pattern it retires is refused** + // **Once the node's tool runtime is in the catalogue, the pattern it retires may not spread** // (novox/hq ADR 0175, to-be 38 WP2.4): a module serving its tools from a container built on the - // runtime's image. Refused at registration, by name, because this is the mechanism that keeps - // the old pattern from returning by habit — a rebuild of an unmoved module stops here with the - // record that says why. Before the runtime exists the pattern is accepted as it always was. + // runtime's image. Refused at registration, by name, for a module that is new to the catalogue + // or that was registered in another shape — the mechanism that keeps the old pattern from + // returning by habit. **Not refused for a module already registered in that shape**: the + // catalogue holds some thirty of them the day the runtime arrives, each moves to a bundle in + // its own change (to-be 38 WP4 onward), and a gate that refused every rebuild of every unmoved + // module in the meantime would stop the whole pipeline to make a point the record already makes. + // Before the runtime exists the pattern is accepted as it always was. if m.Module != catalogue.RuntimeModule { if why := catalogue.ToolContainerOnTheRuntime(m, from.Against); why != "" { runtime, err := i.hasModule(ctx, catalogue.RuntimeModule) @@ -78,7 +82,13 @@ func (i *Inventory) RegisterModule(ctx context.Context, m catalogue.Manifest, fr return err } if runtime { - return fmt.Errorf("%s is not registered: %s", m.Module, why) + already, err := i.registeredInThatShape(ctx, m.Module) + if err != nil { + return err + } + if !already { + return fmt.Errorf("%s is not registered: %s", m.Module, why) + } } } } @@ -110,6 +120,26 @@ func (i *Inventory) RegisterModule(ctx context.Context, m catalogue.Manifest, fr return err } +// registeredInThatShape is whether the catalogue already holds this module as a tools container on +// the runtime's image — judged from the manifest it holds and what that module's newest build stood +// on, the same two things the gate judges a new registration by. False for a module the catalogue +// does not hold. +func (i *Inventory) registeredInThatShape(ctx context.Context, name string) (bool, error) { + held, err := i.Catalogue(ctx) + if err != nil { + return false, err + } + stored, has := held[name] + if !has { + return false, nil + } + against, err := i.BuiltAgainst(ctx) + if err != nil { + return false, err + } + return catalogue.ToolContainerOnTheRuntime(stored, against[name]) != "", nil +} + // hasModule is whether the catalogue holds a module of that name. func (i *Inventory) hasModule(ctx context.Context, name string) (bool, error) { var one int diff --git a/internal/inventory/catalogue_test.go b/internal/inventory/catalogue_test.go index 7940ccb..dbca7af 100644 --- a/internal/inventory/catalogue_test.go +++ b/internal/inventory/catalogue_test.go @@ -688,29 +688,58 @@ func TestRegisteringWithoutProvenanceKeepsTheSeat(t *testing.T) { // Once the node's tool runtime is in the catalogue, a module serving its tools from a container // built on the runtime's image is refused at registration, naming the record (novox/hq ADR 0175, -// to-be 38 WP2.4). Before, it is accepted as it always was — so a mesh converts in the order the -// design says and nothing is refused before there is anything to move to. +// to-be 38 WP2.4) — for a module new to the catalogue or one that had moved away from it; a module +// already standing in that shape is rebuilt as before, so the catalogue's pipeline keeps running +// while each moves (WP3's amendment). Before the runtime, it is accepted as it always was — so a +// mesh converts in the order the design says and nothing is refused before there is anything to +// move to. func TestAToolContainerIsRefusedOnceTheRuntimeIsRegistered(t *testing.T) { inv := fresh(t) + ctx := t.Context() filter := catalogue.Manifest{Module: "nftables", Version: "1", Tools: []string{"firewall_rules"}, Resources: []map[string]any{{"id": "runtime", "type": "container", "name": "mesh-nftables"}}} stoodOn := []string{catalogue.ArtifactStoreScheme + "mesh-tools/runtime@sha256:" + strings.Repeat("d", 64)} - if err := inv.RegisterModule(t.Context(), filter, Source{Repository: "/r", Against: stoodOn}); err != nil { + // Before the runtime exists the old pattern is accepted as it always was — and built, which is + // how the catalogue comes to know what the module stood on. + if err := inv.RegisterModule(ctx, filter, Source{Repository: "/r", Against: stoodOn}); err != nil { t.Fatalf("before the runtime exists the old pattern is accepted: %v", err) } - runtime := catalogue.Manifest{Module: catalogue.RuntimeModule, Version: "1"} - if err := inv.RegisterModule(t.Context(), runtime, Source{Repository: "/r"}); err != nil { + built := aBuild("nf1", "nftables", "") + built.Against = stoodOn + if err := inv.RecordBuild(ctx, built); err != nil { t.Fatal(err) } - err := inv.RegisterModule(t.Context(), filter, Source{Repository: "/r", Against: stoodOn}) - if err == nil || !strings.Contains(err.Error(), "ADR 0175") { - t.Fatalf("the old pattern was registered beside the runtime: %v", err) + runtime := catalogue.Manifest{Module: catalogue.RuntimeModule, Version: "1"} + if err := inv.RegisterModule(ctx, runtime, Source{Repository: "/r"}); err != nil { + t.Fatal(err) } - // A module that moved its tools to a bundle registers. + + // **A module already registered in that shape is rebuilt without complaint** (to-be 38 WP2.4 as + // amended by WP3): some thirty of them stand the day the runtime arrives, and each moves in its + // own change. The gate is against the pattern spreading, not against the pipeline running. + if err := inv.RegisterModule(ctx, filter, Source{Repository: "/r", Against: stoodOn}); err != nil { + t.Fatalf("a rebuild of a module that already had the pattern was refused: %v", err) + } + // A module new to the catalogue in that shape is refused, naming the record. + newcomer := filter + newcomer.Module = "lamp" + err := inv.RegisterModule(ctx, newcomer, Source{Repository: "/r", Against: stoodOn}) + if err == nil || !strings.Contains(err.Error(), "ADR 0175") { + t.Fatalf("a new module in the old pattern was registered beside the runtime: %v", err) + } + // And a module that had moved its tools to a bundle may not come back to a container. moved := filter moved.Resources = nil - if err := inv.RegisterModule(t.Context(), moved, Source{Repository: "/r", Against: stoodOn}); err != nil { + if err := inv.RegisterModule(ctx, moved, Source{Repository: "/r", Against: stoodOn}); err != nil { t.Fatalf("a module whose tools are a bundle was refused: %v", err) } + unbuilt := aBuild("nf2", "nftables", "") + if err := inv.RecordBuild(ctx, unbuilt); err != nil { + t.Fatal(err) + } + err = inv.RegisterModule(ctx, filter, Source{Repository: "/r", Against: stoodOn}) + if err == nil || !strings.Contains(err.Error(), "ADR 0175") { + t.Fatalf("a module that had moved returned to the old pattern unrefused: %v", err) + } }