WP3: a TypeScript bundle carries what it runs with, the runtime's credential belongs to its account, and the gate refuses spreading not standing (hq to-be 38) #226
@@ -968,6 +968,26 @@ func compile(ctx context.Context, run Runner, tree string, chain Toolchain,
|
||||
if _, err := run(ctx, tree, "docker", invocation...); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if chain.Dependencies != "" {
|
||||
// **What the bundle runs with, from the image it was compiled in** (Toolchain.Dependencies).
|
||||
// A second run in the same image rather than a shell wrapped around the compiler: the
|
||||
// compile line stays a plain command a reader can run by hand, and the copy is one more
|
||||
// plain command beside it. Refused by name when the image carries no such directory — an
|
||||
// older toolchain image — because a bundle packed without its dependencies starts nowhere
|
||||
// and says so three layers away from here.
|
||||
copying := []string{
|
||||
"run", "--rm",
|
||||
"--volume", tree + ":" + within,
|
||||
"--workdir", within,
|
||||
base,
|
||||
"sh", "-c",
|
||||
`test -d "$1" || { echo "the toolchain image carries no $1: it predates the mesh shipping a bundle's dependencies, rebuild $2 first" >&2; exit 1; }; cp -a "$1/." "$3/"`,
|
||||
"dependencies", chain.Dependencies, chain.Base, out,
|
||||
}
|
||||
if _, err := run(ctx, tree, "docker", copying...); err != nil {
|
||||
return "", fmt.Errorf("copying the %s dependencies a bundle runs with: %w", chain.Language, err)
|
||||
}
|
||||
}
|
||||
return filepath.Join(tree, out), nil
|
||||
}
|
||||
|
||||
|
||||
@@ -82,6 +82,41 @@ func TestABundleIsCompiledAndPackedWithNoDockerfile(t *testing.T) {
|
||||
if !strings.HasPrefix(digest, "sha256:") {
|
||||
t.Fatalf("the bundle was not pinned: %v", got.Manifest.Resources[0])
|
||||
}
|
||||
|
||||
// **And what it runs with, from the image it was compiled in** (novox/hq to-be 38 WP3). A
|
||||
// second run in the same toolchain image copies the toolchain's runtime directory — the
|
||||
// `"type": "module"` package.json and the pruned node_modules — into the output's root, and
|
||||
// refuses by name when the image carries none rather than packing a bundle that starts nowhere.
|
||||
var copied string
|
||||
for _, line := range r.ran {
|
||||
if strings.HasPrefix(line, "docker run") && strings.Contains(line, "/app/runtime") {
|
||||
copied = line
|
||||
}
|
||||
}
|
||||
if copied == "" {
|
||||
t.Fatalf("the bundle's dependencies were not copied in after the compile:\n%s", strings.Join(r.ran, "\n"))
|
||||
}
|
||||
if !strings.Contains(copied, "mesh-tools/build@sha256:") || !strings.Contains(copied, "predates") ||
|
||||
!strings.Contains(copied, Out("code")) {
|
||||
t.Fatalf("the copy does not run in the same toolchain, refuse an older image by name, or land in the artifact's output: %s", copied)
|
||||
}
|
||||
if strings.Index(strings.Join(r.ran, "\n"), "--outDir") > strings.Index(strings.Join(r.ran, "\n"), "/app/runtime") {
|
||||
t.Fatal("the dependencies were copied before the compile wrote its output")
|
||||
}
|
||||
}
|
||||
|
||||
// A language whose bundle carries its own dependencies copies nothing in: a Go binary is static.
|
||||
func TestOnlyALanguageWithARuntimeDirectoryCopiesDependenciesIn(t *testing.T) {
|
||||
ts, _ := ToolchainFor("typescript")
|
||||
if ts.Dependencies != "/app/runtime" {
|
||||
t.Fatalf("typescript bundles run with %q", ts.Dependencies)
|
||||
}
|
||||
for _, language := range []string{"go", "python"} {
|
||||
chain, _ := ToolchainFor(language)
|
||||
if chain.Dependencies != "" {
|
||||
t.Fatalf("%s copies %q into every bundle, and its bundles carry their own", language, chain.Dependencies)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **Refused before anything is built, naming what to build first.** A base the mesh has not built
|
||||
@@ -145,9 +180,13 @@ func TestTwoBundlesInOneModuleArePackedSeparately(t *testing.T) {
|
||||
t.Fatalf("a module with two bundles did not build: %v", err)
|
||||
}
|
||||
|
||||
// Compiled into two different places.
|
||||
// Compiled into two different places. Only the compile lines: the copy of each bundle's
|
||||
// dependencies names the same directory again, deliberately.
|
||||
var outputs []string
|
||||
for _, line := range r.ran {
|
||||
if !strings.Contains(line, "--outDir") {
|
||||
continue
|
||||
}
|
||||
for _, part := range strings.Fields(line) {
|
||||
if strings.HasPrefix(part, ".mesh-build/") {
|
||||
outputs = append(outputs, part)
|
||||
|
||||
@@ -57,6 +57,23 @@ type Toolchain struct {
|
||||
// carrying its debug info. The mistake was believing a comment rather than reading the file it
|
||||
// produced (novox/hq 04-ISSUES/161).
|
||||
LinkerFlags []string
|
||||
// Dependencies is a directory inside the toolchain image whose contents a bundle in this
|
||||
// language runs with, copied whole into the compiled output's root after the compile.
|
||||
//
|
||||
// **A bundle that compiles is not yet a bundle that runs.** The compiler resolves `import
|
||||
// "nats"` from the toolchain image's own node_modules and the pack takes only what the compiler
|
||||
// wrote, so what a machine unpacked could not find a single dependency — and no TypeScript bundle
|
||||
// had ever run live to show it (novox/hq to-be 38 WP3). For TypeScript the directory holds a
|
||||
// `package.json` saying `"type": "module"` — Node reads a bare `.js` as CommonJS otherwise, so a
|
||||
// bundle with its dependencies and without that line still fails to start — and the pruned,
|
||||
// production-only node_modules the runtime itself ships with: the SDK's and the runtime's
|
||||
// dependencies, and nothing module-specific yet (novox/hq ADR 0188 §5: a skeleton; a module's
|
||||
// own npm dependencies are a later step). Empty for a language whose bundle carries its own —
|
||||
// a Go binary is static, a Python bundle is installed with its dependencies.
|
||||
//
|
||||
// A toolchain image without the directory fails the build by name rather than packing a bundle
|
||||
// that starts nowhere: the image predates this and must be rebuilt first.
|
||||
Dependencies string
|
||||
// SystemStamp is the variable this language's linker fills with the artifact's declared system,
|
||||
// for a language whose binaries are pinned to one at link time (novox/hq ADR 0005).
|
||||
//
|
||||
@@ -118,9 +135,10 @@ var toolchains = []Toolchain{
|
||||
"--module", "NodeNext", "--moduleResolution", "NodeNext",
|
||||
"--target", "ES2022", "--rootDir", ".",
|
||||
},
|
||||
OutputFlag: "--outDir",
|
||||
Unit: UnitSources,
|
||||
SourceExt: ".ts",
|
||||
OutputFlag: "--outDir",
|
||||
Unit: UnitSources,
|
||||
SourceExt: ".ts",
|
||||
Dependencies: "/app/runtime",
|
||||
},
|
||||
{
|
||||
Language: "go",
|
||||
|
||||
@@ -508,7 +508,17 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
|
||||
return nil, fmt.Errorf(
|
||||
"%s needs a secret called %q and none was made for it", m.Module, name)
|
||||
}
|
||||
first = append(first, ownedBy(m.SecretsOwner, map[string]any{
|
||||
// The runtime's credential belongs to the account the runtime runs as (novox/hq ADR 0175,
|
||||
// to-be 38 WP3): its process is composed `user: <account>` where the node has one, and a
|
||||
// root-owned 0600 file is one that process cannot read. Composed here rather than said in
|
||||
// the manifest, because a manifest cannot say ${machine:account} safely — a node with no
|
||||
// account has nothing to resolve it to, and then the runtime runs as root and the file
|
||||
// stays root's.
|
||||
owner := m.SecretsOwner
|
||||
if m.Module == RuntimeModule && r.Account != "" {
|
||||
owner = r.Account
|
||||
}
|
||||
first = append(first, ownedBy(owner, map[string]any{
|
||||
"id": NeedID(name), "type": "file", "path": m.OwnSecrets[name].Path, "sealed": sealed,
|
||||
}))
|
||||
}
|
||||
|
||||
@@ -160,6 +160,11 @@ func TestTheMachineRunsOneRuntimeLoadingEveryDeliveredBundle(t *testing.T) {
|
||||
if env[RuntimeOperatorAccount] != "ops" || env[RuntimeOperatorHome] != "/home/ops" || process["user"] != "ops" {
|
||||
t.Errorf("the operator is not handed to the runtime: %v as %v", env, process["user"])
|
||||
}
|
||||
// The credential the process reads belongs to the account it runs as, or it could not read it
|
||||
// (to-be 38 WP3); other modules' secrets are left as their manifests say.
|
||||
if credential := fileNamed(out, RuntimeModule+"."+NeedID("broker")); credential == nil || credential["owner"] != "ops" {
|
||||
t.Errorf("the runtime's credential is not the account's to read: %v", credential)
|
||||
}
|
||||
restarts := fmt.Sprint(process["restart-on"])
|
||||
for _, want := range []string{"nftables." + BundleID("tools"), "showcase." + BundleID("code"), RuntimeModule + "." + NeedID("broker")} {
|
||||
if !strings.Contains(restarts, want) {
|
||||
@@ -185,6 +190,9 @@ func TestTheMachineRunsOneRuntimeLoadingEveryDeliveredBundle(t *testing.T) {
|
||||
if _, set := process["user"]; set {
|
||||
t.Error("a user was set on a machine with no account")
|
||||
}
|
||||
if credential := fileNamed(out, RuntimeModule+"."+NeedID("broker")); credential == nil || credential["owner"] != nil {
|
||||
t.Errorf("the runtime's credential was given an owner on a machine with no account: %v", credential)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a runtime module built wrong is refused by name", func(t *testing.T) {
|
||||
|
||||
@@ -66,11 +66,15 @@ func (s Source) Current() bool {
|
||||
// gains a requirement, a claim, a resource. What matters is that the change is visible the next
|
||||
// time a node is resolved, which it is.
|
||||
func (i *Inventory) RegisterModule(ctx context.Context, m catalogue.Manifest, from Source) error {
|
||||
// **Once the node's tool runtime is in the catalogue, the pattern it retires is refused**
|
||||
// **Once the node's tool runtime is in the catalogue, the pattern it retires may not spread**
|
||||
// (novox/hq ADR 0175, to-be 38 WP2.4): a module serving its tools from a container built on the
|
||||
// runtime's image. Refused at registration, by name, because this is the mechanism that keeps
|
||||
// the old pattern from returning by habit — a rebuild of an unmoved module stops here with the
|
||||
// record that says why. Before the runtime exists the pattern is accepted as it always was.
|
||||
// runtime's image. Refused at registration, by name, for a module that is new to the catalogue
|
||||
// or that was registered in another shape — the mechanism that keeps the old pattern from
|
||||
// returning by habit. **Not refused for a module already registered in that shape**: the
|
||||
// catalogue holds some thirty of them the day the runtime arrives, each moves to a bundle in
|
||||
// its own change (to-be 38 WP4 onward), and a gate that refused every rebuild of every unmoved
|
||||
// module in the meantime would stop the whole pipeline to make a point the record already makes.
|
||||
// Before the runtime exists the pattern is accepted as it always was.
|
||||
if m.Module != catalogue.RuntimeModule {
|
||||
if why := catalogue.ToolContainerOnTheRuntime(m, from.Against); why != "" {
|
||||
runtime, err := i.hasModule(ctx, catalogue.RuntimeModule)
|
||||
@@ -78,7 +82,13 @@ func (i *Inventory) RegisterModule(ctx context.Context, m catalogue.Manifest, fr
|
||||
return err
|
||||
}
|
||||
if runtime {
|
||||
return fmt.Errorf("%s is not registered: %s", m.Module, why)
|
||||
already, err := i.registeredInThatShape(ctx, m.Module)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !already {
|
||||
return fmt.Errorf("%s is not registered: %s", m.Module, why)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -110,6 +120,26 @@ func (i *Inventory) RegisterModule(ctx context.Context, m catalogue.Manifest, fr
|
||||
return err
|
||||
}
|
||||
|
||||
// registeredInThatShape is whether the catalogue already holds this module as a tools container on
|
||||
// the runtime's image — judged from the manifest it holds and what that module's newest build stood
|
||||
// on, the same two things the gate judges a new registration by. False for a module the catalogue
|
||||
// does not hold.
|
||||
func (i *Inventory) registeredInThatShape(ctx context.Context, name string) (bool, error) {
|
||||
held, err := i.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
stored, has := held[name]
|
||||
if !has {
|
||||
return false, nil
|
||||
}
|
||||
against, err := i.BuiltAgainst(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return catalogue.ToolContainerOnTheRuntime(stored, against[name]) != "", nil
|
||||
}
|
||||
|
||||
// hasModule is whether the catalogue holds a module of that name.
|
||||
func (i *Inventory) hasModule(ctx context.Context, name string) (bool, error) {
|
||||
var one int
|
||||
|
||||
@@ -688,29 +688,58 @@ func TestRegisteringWithoutProvenanceKeepsTheSeat(t *testing.T) {
|
||||
|
||||
// Once the node's tool runtime is in the catalogue, a module serving its tools from a container
|
||||
// built on the runtime's image is refused at registration, naming the record (novox/hq ADR 0175,
|
||||
// to-be 38 WP2.4). Before, it is accepted as it always was — so a mesh converts in the order the
|
||||
// design says and nothing is refused before there is anything to move to.
|
||||
// to-be 38 WP2.4) — for a module new to the catalogue or one that had moved away from it; a module
|
||||
// already standing in that shape is rebuilt as before, so the catalogue's pipeline keeps running
|
||||
// while each moves (WP3's amendment). Before the runtime, it is accepted as it always was — so a
|
||||
// mesh converts in the order the design says and nothing is refused before there is anything to
|
||||
// move to.
|
||||
func TestAToolContainerIsRefusedOnceTheRuntimeIsRegistered(t *testing.T) {
|
||||
inv := fresh(t)
|
||||
ctx := t.Context()
|
||||
filter := catalogue.Manifest{Module: "nftables", Version: "1", Tools: []string{"firewall_rules"},
|
||||
Resources: []map[string]any{{"id": "runtime", "type": "container", "name": "mesh-nftables"}}}
|
||||
stoodOn := []string{catalogue.ArtifactStoreScheme + "mesh-tools/runtime@sha256:" + strings.Repeat("d", 64)}
|
||||
|
||||
if err := inv.RegisterModule(t.Context(), filter, Source{Repository: "/r", Against: stoodOn}); err != nil {
|
||||
// Before the runtime exists the old pattern is accepted as it always was — and built, which is
|
||||
// how the catalogue comes to know what the module stood on.
|
||||
if err := inv.RegisterModule(ctx, filter, Source{Repository: "/r", Against: stoodOn}); err != nil {
|
||||
t.Fatalf("before the runtime exists the old pattern is accepted: %v", err)
|
||||
}
|
||||
runtime := catalogue.Manifest{Module: catalogue.RuntimeModule, Version: "1"}
|
||||
if err := inv.RegisterModule(t.Context(), runtime, Source{Repository: "/r"}); err != nil {
|
||||
built := aBuild("nf1", "nftables", "")
|
||||
built.Against = stoodOn
|
||||
if err := inv.RecordBuild(ctx, built); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err := inv.RegisterModule(t.Context(), filter, Source{Repository: "/r", Against: stoodOn})
|
||||
if err == nil || !strings.Contains(err.Error(), "ADR 0175") {
|
||||
t.Fatalf("the old pattern was registered beside the runtime: %v", err)
|
||||
runtime := catalogue.Manifest{Module: catalogue.RuntimeModule, Version: "1"}
|
||||
if err := inv.RegisterModule(ctx, runtime, Source{Repository: "/r"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A module that moved its tools to a bundle registers.
|
||||
|
||||
// **A module already registered in that shape is rebuilt without complaint** (to-be 38 WP2.4 as
|
||||
// amended by WP3): some thirty of them stand the day the runtime arrives, and each moves in its
|
||||
// own change. The gate is against the pattern spreading, not against the pipeline running.
|
||||
if err := inv.RegisterModule(ctx, filter, Source{Repository: "/r", Against: stoodOn}); err != nil {
|
||||
t.Fatalf("a rebuild of a module that already had the pattern was refused: %v", err)
|
||||
}
|
||||
// A module new to the catalogue in that shape is refused, naming the record.
|
||||
newcomer := filter
|
||||
newcomer.Module = "lamp"
|
||||
err := inv.RegisterModule(ctx, newcomer, Source{Repository: "/r", Against: stoodOn})
|
||||
if err == nil || !strings.Contains(err.Error(), "ADR 0175") {
|
||||
t.Fatalf("a new module in the old pattern was registered beside the runtime: %v", err)
|
||||
}
|
||||
// And a module that had moved its tools to a bundle may not come back to a container.
|
||||
moved := filter
|
||||
moved.Resources = nil
|
||||
if err := inv.RegisterModule(t.Context(), moved, Source{Repository: "/r", Against: stoodOn}); err != nil {
|
||||
if err := inv.RegisterModule(ctx, moved, Source{Repository: "/r", Against: stoodOn}); err != nil {
|
||||
t.Fatalf("a module whose tools are a bundle was refused: %v", err)
|
||||
}
|
||||
unbuilt := aBuild("nf2", "nftables", "")
|
||||
if err := inv.RecordBuild(ctx, unbuilt); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = inv.RegisterModule(ctx, filter, Source{Repository: "/r", Against: stoodOn})
|
||||
if err == nil || !strings.Contains(err.Error(), "ADR 0175") {
|
||||
t.Fatalf("a module that had moved returned to the old pattern unrefused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user