Merge pull request 'A tools bundle is given its words, composed per machine; what they name is the account's to read (hq ADR 0192)' (#236) from feat/0192-a-bundles-env into main

internal/catalogue/build.go

@@ -92,7 +92,7 @@ func (m Manifest) Resolve(built []Built) (Manifest, error) {
 			out.Bundles = append(out.Bundles, Bundle{
 				Name: a.Name, Source: Recorded(made.Reference), Digest: made.Digest,
 				Language: a.Language, Entrypoints: append([]string(nil), a.Entrypoints...),
-				Loads: loads,
+				Loads: loads, Env: copyWords(a.Env),
 			})
 		}
 		sort.Slice(out.Bundles, func(i, j int) bool { return out.Bundles[i].Name < out.Bundles[j].Name })
@@ -203,6 +203,12 @@ func (b *Build) problems(module string) []string {
 		// A bundle's source is the module's own directory by definition, and what it needs to say
 		// is which compiler — because the mesh chooses that, and cannot choose for a module that
 		// has not said.
+		if len(a.Env) > 0 && a.Kind != ArtifactBundle {
+			problems = append(problems, fmt.Sprintf(
+				"%s: %q is a %q and says what it is given (env). Only a bundle the node's runtime "+
+					"serves is given words (novox/hq ADR 0192); a container says its own environment",
+				module, a.Name, a.Kind))
+		}
 		if a.Kind == ArtifactBundle || a.Kind == ArtifactPackage {
 			// **Except for a language that compiles to a binary, where it names which one**
 			// (novox/hq 04-ISSUES/142). A bundle in an interpreted language is the module's own
@@ -222,6 +228,7 @@ func (b *Build) problems(module string) []string {
 					"%s: %q is a bundle and says no language, so nothing can choose a compiler "+
 						"for it", module, a.Name))
 			}
+			problems = append(problems, bundleEnvProblems(module, a)...)
 			// What the runtime loads is among what was compiled (ADR 0175): a name here that is
 			// not an entrypoint is a file the bundle does not contain, and the runtime would
 			// fail to import it on every machine rather than here.
@@ -360,3 +367,50 @@ func versionOf(digest string) string {
 	}
 	return hex
 }
+
+// bundleEnvWords are the words the runtime sets for itself; a bundle that named one would be
+// telling the runtime what it is, which is the mesh's to say (novox/hq ADR 0192).
+var bundleEnvWords = map[string]bool{
+	RuntimeToolModules: true, RuntimeBrokerFile: true, RuntimeOperatorAccount: true,
+	RuntimeOperatorHome: true, RuntimeToolEnv: true,
+}
+
+// bundleEnvProblems says what is wrong with what a bundle says it is given (novox/hq ADR 0192):
+// a value is a path or a constant written with the references a container's environment may use
+// for a place or a port, and never a secret's content or another module's binding — a secret
+// reaches a tool as a file whose path is named.
+func bundleEnvProblems(module string, a Artifact) []string {
+	if len(a.Env) == 0 {
+		return nil
+	}
+	var problems []string
+	for _, word := range sortedKeys(a.Env) {
+		value := a.Env[word]
+		if bundleEnvWords[word] {
+			problems = append(problems, fmt.Sprintf(
+				"%s: %q gives itself %s, which the node's runtime sets for itself; a bundle is "+
+					"given its own words beside the runtime's, never in place of them (novox/hq ADR 0192)",
+				module, a.Name, word))
+		}
+		rest := ofPort.ReplaceAllString(dirRef.ReplaceAllString(value, ""), "")
+		if strings.Contains(rest, "${") {
+			problems = append(problems, fmt.Sprintf(
+				"%s: %q gives %s the value %q. A bundle's word is a path or a constant, written with "+
+					"${dir:…} and ${port:…} only; a secret reaches a tool as a file the mesh places, "+
+					"named by its path, never as its content (novox/hq ADR 0192)",
+				module, a.Name, word, value))
+		}
+	}
+	return problems
+}
+
+func copyWords(in map[string]string) map[string]string {
+	if len(in) == 0 {
+		return nil
+	}
+	out := make(map[string]string, len(in))
+	for k, v := range in {
+		out[k] = v
+	}
+	return out
+}

internal/catalogue/declaration.go

@@ -921,6 +921,16 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
 		}
 		owner[fmt.Sprint(process["id"])] = RuntimeModule
 		out = append(out, process)
+		// What each module's bundles are given is read as the account the runtime runs as.
+		words := map[string]map[string]string{}
+		for _, m := range r.Modules {
+			w, err := bundleWords(m, with)
+			if err != nil {
+				return nil, err
+			}
+			words[m.Module] = w
+		}
+		givenTo(out, owner, words, r.Account)
 	}
 	if with.Adopted {
 		// First, before anything a module declares: what the mesh needs reachable, then its guard.

internal/catalogue/manifest.go

@@ -602,6 +602,8 @@ type Bundle struct {
 	// every entrypoint for a module declaring tools that said nothing. Empty for a bundle that is
 	// run rather than loaded.
 	Loads []string `json:"loads,omitempty"`
+	// Env is what the artifact said it is given (ADR 0192), as written; composed per machine.
+	Env map[string]string `json:"env,omitempty"`
 }
 
 // Build says how to produce this module's artifacts from its source.
@@ -748,6 +750,11 @@ type Artifact struct {
 	// module's tools and nothing else is the ordinary case and should not have to say the same
 	// list twice. A module declaring no tools has nothing the runtime loads, whatever it compiles.
 	Loads []string `json:"loads,omitempty"`
+
+	// Env is what a tools bundle is given on a machine (novox/hq ADR 0192): words and their values,
+	// paths and constants composed with ${dir:…} and ${port:…} exactly as a container's environment
+	// is, never a secret's content. The node's runtime hands it to this bundle and to no other.
+	Env map[string]string `json:"env,omitempty"`
 }
 
 // Kinds an artifact may be.

internal/catalogue/runtime.go

@@ -1,6 +1,7 @@
 package catalogue
 
 import (
+	"encoding/json"
 	"fmt"
 	"sort"
 	"strings"
@@ -82,6 +83,11 @@ const (
 	RuntimeBrokerFile      = "MESH_BROKER_FILE"
 	RuntimeOperatorAccount = "MESH_OPERATOR_ACCOUNT"
 	RuntimeOperatorHome    = "MESH_OPERATOR_HOME"
+	// RuntimeToolEnv is every served module's composed environment, as JSON (novox/hq ADR 0192):
+	// {"<module>": {"<word>": "<value>"}}. The runtime takes it at start, removes it from its own
+	// environment and hands each module's words to that module's bundles alone. In the unit, so a
+	// change to any module's words changes the process and restarts it.
+	RuntimeToolEnv = "MESH_TOOL_ENV"
 )
 
 // interpreterFor is how a bundle in a language is run: the program the host's unit starts, with the
@@ -146,10 +152,18 @@ func (r Resolution) runtimeProcess(with Rendering) (map[string]any, error) {
 	// would be told to load files that were never delivered.
 	var served []string
 	var restartOn []string
+	given := map[string]map[string]string{}
 	for _, m := range r.Modules {
 		if with.Adopted && m.Filtering != nil {
 			continue
 		}
+		words, err := bundleWords(m, with)
+		if err != nil {
+			return nil, err
+		}
+		if len(words) > 0 {
+			given[m.Module] = words
+		}
 		for _, b := range m.Bundles {
 			if len(b.Loads) == 0 {
 				continue
@@ -168,6 +182,14 @@ func (r Resolution) runtimeProcess(with Rendering) (map[string]any, error) {
 		RuntimeToolModules: strings.Join(served, ","),
 		RuntimeBrokerFile:  credential.Path,
 	}
+	if len(given) > 0 {
+		// Marshalled from maps, whose keys encoding/json sorts: the same words, the same unit.
+		body, err := json.Marshal(given)
+		if err != nil {
+			return nil, err
+		}
+		env[RuntimeToolEnv] = string(body)
+	}
 	process := map[string]any{
 		"id": RuntimeModule + "." + RuntimeProcessID(), "type": "process", "name": RuntimeModule,
 		"source": bundle.Source, "digest": bundle.Digest,
@@ -249,3 +271,71 @@ func ToolContainerOnTheRuntime(m Manifest, against []string) string {
 			"now (novox/hq ADR 0175, to-be 38); declare the tools as a bundle and drop the container",
 		m.Module, RuntimeImageModule, RuntimeImageArtifact, RuntimeModule)
 }
+
+// bundleWords is what one module's tools bundles are given on this machine (novox/hq ADR 0192):
+// each loaded bundle's env, its ${dir:…} resolved to where this machine places the module's
+// directories and its ${port:…} to the port this machine gave it — the same resolution a
+// container's environment gets. Two bundles of one module naming one word differently is refused:
+// the runtime hands a module's words to all its bundles.
+func bundleWords(m Manifest, with Rendering) (map[string]string, error) {
+	var out map[string]string
+	dirs := dirsFor(m, with)
+	for _, b := range m.Bundles {
+		if len(b.Loads) == 0 || len(b.Env) == 0 {
+			continue
+		}
+		for _, word := range sortedKeys(b.Env) {
+			value, err := dirFill(b.Env[word], dirs, m.Module)
+			if err != nil {
+				return nil, err
+			}
+			if value, err = portsFilledInto(value, m.Module+"'s bundle "+b.Name+" ("+word+")", m.Module, m.Listens, with); err != nil {
+				return nil, err
+			}
+			if out == nil {
+				out = map[string]string{}
+			}
+			if was, had := out[word]; had && was != value {
+				return nil, fmt.Errorf("%s's bundles give %s two values (%q, %q); a module's words are "+
+					"handed to all its bundles, so they agree (novox/hq ADR 0192)", m.Module, word, was, value)
+			}
+			out[word] = value
+		}
+	}
+	return out, nil
+}
+
+// givenTo makes what a bundle's words name readable by the account the runtime runs as (novox/hq
+// ADR 0192): every file and directory of the module whose path a word names, or that holds one,
+// is owned by the account — a tool reads its configuration and its secret as the account, and a
+// root-owned 0600 file or a 0700 directory is one it cannot. Only where it says no owner already:
+// a module that named one knew why. Nothing on a machine with no account, where the runtime is root.
+func givenTo(out []map[string]any, owner map[string]string, words map[string]map[string]string, account string) {
+	if account == "" || len(words) == 0 {
+		return
+	}
+	for _, resource := range out {
+		module := owner[fmt.Sprint(resource["id"])]
+		mine := words[module]
+		if len(mine) == 0 {
+			continue
+		}
+		kind := fmt.Sprint(resource["type"])
+		if kind != "file" && kind != "directory" {
+			continue
+		}
+		path, _ := resource["path"].(string)
+		if path == "" {
+			continue
+		}
+		if _, said := resource["owner"]; said {
+			continue
+		}
+		for _, value := range mine {
+			if value == path || strings.HasPrefix(value, strings.TrimRight(path, "/")+"/") {
+				resource["owner"] = account
+				break
+			}
+		}
+	}
+}

internal/catalogue/runtime_test.go

@@ -1,6 +1,7 @@
 package catalogue
 
 import (
+	"encoding/json"
 	"fmt"
 	"strings"
 	"testing"
@@ -210,3 +211,118 @@ func TestTheMachineRunsOneRuntimeLoadingEveryDeliveredBundle(t *testing.T) {
 		}
 	})
 }
+
+// novox/hq ADR 0192: a tools bundle says what it is given; the composer resolves it per machine as
+// a container's environment, hands it to the runtime as the module's words, and makes what the
+// words name readable by the account the runtime runs as.
+func TestABundleIsGivenItsWordsResolvedForThisMachine(t *testing.T) {
+	with := Rendering{ArtifactStore: "anchor.internal:5101",
+		Needed: map[string]map[string]string{
+			RuntimeModule: {"broker": "sealed-credential"},
+			"dash":        {"token": "sealed-token"},
+		}}
+	dash := Manifest{Module: "dash", Version: "1", Tools: []string{"status"},
+		Listens:    []Listening{{Name: "web", Port: 3000, Protocol: "tcp"}},
+		OwnSecrets: OwnSecrets{"token": {Path: "${dir:mesh-state}/token"}},
+		Resources: []map[string]any{
+			{"id": "mesh-state", "type": "directory", "mode": "0700", "place": "mesh"},
+			{"id": "config", "type": "file", "path": "${dir:mesh-state}/config.json", "mode": "0600", "content": "{}\n"},
+			{"id": "unrelated", "type": "file", "path": "/etc/dash.conf", "content": "x\n"},
+		},
+		Build: &Build{Artifacts: []Artifact{{Name: "tools", Kind: ArtifactBundle, Language: "typescript",
+			Entrypoints: []string{"tools/index.js"},
+			Env: map[string]string{
+				"DASH_CONFIG_FILE": "${dir:mesh-state}/config.json",
+				"DASH_TOKEN_FILE":  "${dir:mesh-state}/token",
+				"DASH_URL":         "http://127.0.0.1:${port:3000}",
+				"DASH_ADMIN":       "mesh-admin",
+			}}}}}
+	if problems := dash.Build.problems(dash.Module); len(problems) > 0 {
+		t.Fatalf("a bundle's words written with ${dir:…} and ${port:…} were refused: %v", problems)
+	}
+	dash, err := dash.Resolve([]Built{{Name: "tools", Kind: ArtifactBundle,
+		Reference: ArtifactStoreScheme + "dash/tools/blobs/" + bundleDigest, Digest: bundleDigest}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	other := aToolsModule(t, "nftables", "tools/index.js")
+
+	out, err := Resolution{Node: "anchor", Account: "ops", Modules: []Manifest{dash, other, theRuntime(t)}}.Declaration(with)
+	if err != nil {
+		t.Fatal(err)
+	}
+	dir := fileNamed(out, "dash.mesh-state")
+	if dir == nil {
+		t.Fatalf("no directory: %v", ids(out))
+	}
+	at := fmt.Sprint(dir["path"])
+	process := fileNamed(out, RuntimeModule+"."+RuntimeProcessID())
+	env := process["env"].(map[string]string)
+	var given map[string]map[string]string
+	if err := json.Unmarshal([]byte(env[RuntimeToolEnv]), &given); err != nil {
+		t.Fatalf("the runtime's %s is not JSON: %q", RuntimeToolEnv, env[RuntimeToolEnv])
+	}
+	want := map[string]string{
+		"DASH_CONFIG_FILE": at + "/config.json",
+		"DASH_TOKEN_FILE":  at + "/token",
+		"DASH_URL":         "http://127.0.0.1:3000",
+		"DASH_ADMIN":       "mesh-admin",
+	}
+	if fmt.Sprint(given["dash"]) != fmt.Sprint(want) {
+		t.Errorf("dash is given %v, want %v", given["dash"], want)
+	}
+	if _, has := given["nftables"]; has {
+		t.Errorf("a module that declares no words was given some: %v", given)
+	}
+	// What the words name is the account's to read; nothing else of the module's is touched.
+	for _, id := range []string{"dash.mesh-state", "dash.config", "dash." + NeedID("token")} {
+		if r := fileNamed(out, id); r == nil || r["owner"] != "ops" {
+			t.Errorf("%s is not the account's to read: %v", id, r)
+		}
+	}
+	if r := fileNamed(out, "dash.unrelated"); r == nil || r["owner"] != nil {
+		t.Errorf("a file no word names was given an owner: %v", r)
+	}
+
+	t.Run("on a machine with no account the runtime is root and nothing is re-owned", func(t *testing.T) {
+		out, err := Resolution{Node: "anchor", Modules: []Manifest{dash, theRuntime(t)}}.Declaration(with)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if r := fileNamed(out, "dash.config"); r["owner"] != nil {
+			t.Errorf("re-owned with no account: %v", r)
+		}
+	})
+
+	t.Run("a change to a module's words changes the runtime's process", func(t *testing.T) {
+		changed := dash
+		changed.Bundles = append([]Bundle(nil), dash.Bundles...)
+		changed.Bundles[0].Env = map[string]string{"DASH_ADMIN": "somebody-else"}
+		out2, err := Resolution{Node: "anchor", Account: "ops", Modules: []Manifest{changed, theRuntime(t)}}.Declaration(with)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if fmt.Sprint(fileNamed(out2, RuntimeModule+"."+RuntimeProcessID())["env"]) == fmt.Sprint(env) {
+			t.Error("the runtime's process is the same after a module's words changed, so it would not restart")
+		}
+	})
+}
+
+func TestABundlesWordsAreRefusedWhenTheyAreNotPathsOrConstants(t *testing.T) {
+	m := Manifest{Module: "dash", Version: "1", Build: &Build{Artifacts: []Artifact{
+		{Name: "tools", Kind: ArtifactBundle, Language: "typescript", Entrypoints: []string{"tools/index.js"},
+			Env: map[string]string{"DASH_TOKEN": "${secret:token}", RuntimeBrokerFile: "/x", "DASH_PEER": "${bound:db:url}"}},
+		{Name: "runtime", Kind: ArtifactImage, From: "Dockerfile", Env: map[string]string{"X": "y"}},
+	}}}
+	said := strings.Join(m.Build.problems(m.Module), "\n")
+	for _, want := range []string{
+		`"tools" gives DASH_TOKEN the value "${secret:token}"`,
+		`"tools" gives DASH_PEER the value "${bound:db:url}"`,
+		`"tools" gives itself ` + RuntimeBrokerFile,
+		`"runtime" is a "image" and says what it is given`,
+	} {
+		if !strings.Contains(said, want) {
+			t.Errorf("not refused: %s\nsaid:\n%s", want, said)
+		}
+	}
+}