Give mesh-sdk a composer command, so a program linking one mesh-sdk can have another commit's composition judged (issue 1621)

2026-10-24 03:51:35 +02:00
parent e1f393bbb1
commit 20d18eb4f5
4 changed files with 875 additions and 0 deletions
@@ -0,0 +1,144 @@
+// mesh-compose is mesh-sdk's composer command (novox/hq ADR 0482 clause 4, issue 1621): it reads a composer.Request on
+// standard input, validates every manifest with this mesh-sdk's reader, composes the machines named in each case and
+// writes one composer.Verdict on standard output. Built at any commit of mesh-sdk (or at the mesh-sdk a module pins),
+// it answers what that commit's composition answers, so a program that links one mesh-sdk can have another's judged.
+//
+//	mesh-compose < request.json > verdict.json
+//	mesh-compose -catalogue <dir> [-node <name>]... [-depth resolve|compose] [-declarations] [< request.json]
+//
+// -catalogue reads <dir>/modules/*/module.json and a module.json at <dir>'s root as the first case's manifests (a
+// checkout or `git archive` of the catalogue repository at a ref: the command fetches nothing). -node adds a machine to
+// compose; -declarations answers each machine's declared paths with their owners and modes.
+//
+// It opens no network connection, bus, store or file beyond its standard input and the -catalogue directory, and reads
+// no environment. Exit codes: 0 the verdict is written (a machine that does not compose is a verdict, not an error);
+// 1 the request cannot be read or run, said on standard error; 2 the flags are wrong.
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"git.novox.be/novox/mesh-sdk/go/composer"
+)
+
+func main() {
+	os.Exit(run(os.Args[1:], os.Stdin, os.Stdout, os.Stderr))
+}
+
+type nodes []string
+
+func (n *nodes) String() string     { return strings.Join(*n, ",") }
+func (n *nodes) Set(v string) error { *n = append(*n, v); return nil }
+
+// run is the command: its arguments, standard input, output and error; it answers the exit code.
+func run(args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("mesh-compose", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	catalogueDir := fs.String("catalogue", "", "a catalogue checkout: <dir>/modules/*/module.json are the first case's manifests")
+	var machines nodes
+	fs.Var(&machines, "node", "a machine to compose (repeatable)")
+	depth := fs.String("depth", "", "resolve or compose (default: the request's, else compose)")
+	declarations := fs.Bool("declarations", false, "answer each composed machine's declared paths, owners and modes")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	if fs.NArg() > 0 {
+		fmt.Fprintf(stderr, "mesh-compose: unexpected arguments %q\n", fs.Args())
+		return 2
+	}
+	raw, err := io.ReadAll(stdin)
+	if err != nil {
+		fmt.Fprintln(stderr, "mesh-compose: standard input cannot be read:", err)
+		return 1
+	}
+	var req composer.Request
+	if strings.TrimSpace(string(raw)) == "" {
+		if *catalogueDir == "" {
+			fmt.Fprintln(stderr, "mesh-compose: no request on standard input and no -catalogue")
+			return 1
+		}
+		req = composer.Request{V: composer.Version}
+	} else if req, err = composer.Decode(raw); err != nil {
+		fmt.Fprintln(stderr, "mesh-compose:", err)
+		return 1
+	}
+	if *catalogueDir != "" {
+		read, err := readCatalogue(*catalogueDir)
+		if err != nil {
+			fmt.Fprintln(stderr, "mesh-compose:", err)
+			return 1
+		}
+		if len(req.Cases) == 0 {
+			req.Cases = []composer.Case{{Name: "catalogue", Manifests: map[string]composer.Manifest{}}}
+		}
+		if req.Cases[0].Manifests == nil {
+			req.Cases[0].Manifests = map[string]composer.Manifest{}
+		}
+		for name, m := range read {
+			if _, given := req.Cases[0].Manifests[name]; !given {
+				req.Cases[0].Manifests[name] = m
+			}
+		}
+	}
+	req.Machines = append(req.Machines, machines...)
+	if *depth != "" {
+		req.Depth = *depth
+	}
+	if *declarations {
+		req.Declarations = true
+	}
+	v, err := composer.Run(req)
+	if err != nil {
+		fmt.Fprintln(stderr, "mesh-compose:", err)
+		return 1
+	}
+	enc := json.NewEncoder(stdout)
+	if err := enc.Encode(v); err != nil {
+		fmt.Fprintln(stderr, "mesh-compose: the verdict cannot be written:", err)
+		return 1
+	}
+	return 0
+}
+
+// readCatalogue reads every module.json of a catalogue checkout by its module's name: the root's and each of
+// modules/*/. A manifest whose name cannot be read is kept under its directory's name, so the reader's refusal is said.
+func readCatalogue(dir string) (map[string]composer.Manifest, error) {
+	paths, err := filepath.Glob(filepath.Join(dir, "modules", "*", "module.json"))
+	if err != nil {
+		return nil, err
+	}
+	if _, err := os.Stat(filepath.Join(dir, "module.json")); err == nil {
+		paths = append(paths, filepath.Join(dir, "module.json"))
+	}
+	if len(paths) == 0 {
+		return nil, fmt.Errorf("the catalogue %s holds no module.json", dir)
+	}
+	sort.Strings(paths)
+	out := map[string]composer.Manifest{}
+	for _, p := range paths {
+		raw, err := os.ReadFile(p)
+		if err != nil {
+			return nil, err
+		}
+		var probe struct {
+			Module string `json:"module"`
+		}
+		name := filepath.Base(filepath.Dir(p))
+		if json.Unmarshal(raw, &probe) == nil && probe.Module != "" {
+			name = probe.Module
+		}
+		if _, twice := out[name]; twice {
+			return nil, errors.New("the catalogue " + dir + " defines " + name + " twice")
+		}
+		out[name] = composer.Manifest{Raw: raw}
+	}
+	return out, nil
+}
@@ -0,0 +1,293 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"go/build"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"reflect"
+	"runtime"
+	"sort"
+	"strings"
+	"testing"
+
+	"git.novox.be/novox/mesh-sdk/go/catalogue"
+	"git.novox.be/novox/mesh-sdk/go/compose"
+	"git.novox.be/novox/mesh-sdk/go/composer"
+	"git.novox.be/novox/mesh-sdk/go/internal/beside"
+)
+
+// captured is every manifest of the catalogue captured in testdata/beside (internal/beside), raw, by module name.
+func captured(t *testing.T) map[string]composer.Manifest {
+	t.Helper()
+	read, err := readCatalogue(filepath.Dir(beside.Catalogue(t)))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(read) < 100 {
+		t.Fatalf("the captured catalogue holds %d manifests, fewer than the catalogue's", len(read))
+	}
+	return read
+}
+
+// meshOf is a mesh of three machines over the captured catalogue, each assigned real modules, the input the assigner
+// composes with (nodes and their assignments, holdings from the claims): anchor runs the bus and the store, laptop a
+// desktop's modules, ace a module whose requirement is provided elsewhere.
+func meshOf() compose.Input {
+	return compose.Input{Nodes: []compose.Node{
+		{Name: "anchor", Assigned: []string{"nats", "postgres"}},
+		{Name: "laptop", Assigned: []string{"sudo", "zsh"}},
+		{Name: "ace", Assigned: []string{"mesh-reads", "nats"}},
+	}}
+}
+
+func request(t *testing.T, depth string) composer.Request {
+	t.Helper()
+	return composer.Request{V: composer.Version, Input: meshOf(), Depth: depth, HoldingsFromClaims: true,
+		Machines: []string{"anchor", "laptop", "ace"},
+		Cases:    []composer.Case{{Name: "trunk", Manifests: captured(t)}}}
+}
+
+func runJSON(t *testing.T, args []string, in []byte) (int, composer.Verdict, string) {
+	t.Helper()
+	var out, errOut bytes.Buffer
+	code := run(args, bytes.NewReader(in), &out, &errOut)
+	var v composer.Verdict
+	if code == 0 {
+		if err := json.Unmarshal(out.Bytes(), &v); err != nil {
+			t.Fatalf("the verdict is not JSON: %v\n%s", err, out.String())
+		}
+	}
+	return code, v, errOut.String()
+}
+
+func encode(t *testing.T, r composer.Request) []byte {
+	t.Helper()
+	raw, err := json.Marshal(r)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return raw
+}
+
+// 1621/AC1: the command reads the plain input and the manifests on standard input, validates every manifest, composes
+// the machines named and writes one verdict: per machine composes or the reason, per manifest accepted or the field
+// refused.
+func TestTheCommandWritesOneVerdictPerManifestAndMachine(t *testing.T) {
+	r := request(t, composer.DepthResolve)
+	r.Cases[0].Manifests["broken"] = composer.Manifest{Raw: json.RawMessage(`{"module":"broken","version":"1","shares":["x"]}`)}
+	code, v, stderr := runJSON(t, nil, encode(t, r))
+	if code != 0 {
+		t.Fatalf("exit %d: %s", code, stderr)
+	}
+	if len(v.Cases) != 1 || v.Cases[0].Name != "trunk" || v.SDK == "" {
+		t.Fatalf("verdict = %+v", v)
+	}
+	c := v.Cases[0]
+	if got := c.Manifests["broken"]; got.Accepted || !strings.Contains(got.Refused, `"shares"`) {
+		t.Errorf("the manifest with an unknown field reads %+v, want refused naming the field", got)
+	}
+	if got := c.Manifests["nats"]; !got.Accepted {
+		t.Errorf("nats reads %+v, want accepted", got)
+	}
+	if len(c.Machines) != 3 {
+		t.Fatalf("machines = %v, want the three named", c.Machines)
+	}
+	for name, m := range c.Machines {
+		if !m.Composes && m.Reason == "" {
+			t.Errorf("%s does not compose and says no reason", name)
+		}
+	}
+}
+
+// 1621/AC3: over the captured catalogue and a mesh of three machines, the command's verdict equals, machine for
+// machine and manifest for manifest, what the linked reader, resolution and composition answer, at both depths.
+func TestTheCommandAnswersWhatTheLinkedCompositionAnswers(t *testing.T) {
+	for _, depth := range []string{composer.DepthResolve, composer.DepthCompose} {
+		r := request(t, depth)
+		code, v, stderr := runJSON(t, nil, encode(t, r))
+		if code != 0 {
+			t.Fatalf("%s: exit %d: %s", depth, code, stderr)
+		}
+		in := meshOf()
+		in.Catalogue = map[string]catalogue.Manifest{}
+		for name, m := range r.Cases[0].Manifests {
+			parsed, err := catalogue.ParseManifest(m.Raw)
+			got := v.Cases[0].Manifests[name]
+			if (err == nil) != got.Accepted || (err != nil && err.Error() != got.Refused) {
+				t.Errorf("%s: the manifest of %s: linked %v, command %+v", depth, name, err, got)
+			}
+			if err == nil {
+				in.Catalogue[name] = parsed
+			}
+		}
+		in.Holdings = composer.HoldingsOf(in.Catalogue, in.Nodes)
+		composed := 0
+		for _, node := range r.Machines {
+			got := v.Cases[0].Machines[node]
+			var want composer.Machine
+			if depth == composer.DepthResolve {
+				if _, err := compose.Resolve(in, node); err != nil {
+					want = composer.Machine{Reason: err.Error()}
+				} else {
+					want = composer.Machine{Composes: true}
+				}
+			} else {
+				d, problems, err := compose.Compose(in, node)
+				switch {
+				case err != nil:
+					want = composer.Machine{Reason: err.Error()}
+				default:
+					want = composer.Machine{Composes: len(problems) == 0, Problems: problems, Refusals: d.Refusals()}
+					if len(problems) > 0 {
+						want.Reason = "the node-engine's validator refuses the declaration"
+					}
+				}
+			}
+			if !reflect.DeepEqual(got, want) {
+				t.Errorf("%s: %s: command %+v, linked %+v", depth, node, got, want)
+			}
+			if got.Composes {
+				composed++
+			} else {
+				t.Logf("%s: %s does not compose: %s %v", depth, node, got.Reason, got.Problems)
+			}
+		}
+		t.Logf("%s: %d of %d machines compose", depth, composed, len(r.Machines))
+	}
+}
+
+// 1621 note 6: a person composes a catalogue checkout (repository@ref checked out or archived: the command fetches
+// nothing) for a node and reads its declaration: each path with its kind, owner and mode.
+func TestACatalogueCheckoutIsComposedForANode(t *testing.T) {
+	dir := t.TempDir()
+	if err := os.MkdirAll(filepath.Join(dir, "modules", "demo"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	manifest := `{"module": "demo", "version": "1", "resources": [{"id": "state", "type": "directory", ` +
+		`"path": "/var/lib/demo", "mode": "0750", "owner": "root"}]}`
+	if err := os.WriteFile(filepath.Join(dir, "modules", "demo", "module.json"), []byte(manifest), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	in := encode(t, composer.Request{V: composer.Version, Input: compose.Input{Nodes: []compose.Node{{Name: "laptop", Assigned: []string{"demo"}}}}})
+	code, v, stderr := runJSON(t, []string{"-catalogue", dir, "-node", "laptop", "-declarations"}, in)
+	if code != 0 {
+		t.Fatalf("exit %d: %s", code, stderr)
+	}
+	m := v.Cases[0].Machines["laptop"]
+	want := []composer.Path{{Path: "/var/lib/demo", Kind: "directory", Owner: "root", Mode: "0750", Module: ""}}
+	if !m.Composes || len(m.Declared) != 1 || m.Declared[0].Path != want[0].Path || m.Declared[0].Mode != "0750" ||
+		m.Declared[0].Owner != "root" || m.Declared[0].Kind != "directory" {
+		t.Fatalf("laptop = %+v, want it composed declaring %+v", m, want)
+	}
+}
+
+// The exit codes: 1 for a request that cannot be read or run, 2 for wrong flags, each said on standard error.
+func TestTheExitCodesSayWhatFailed(t *testing.T) {
+	unfilled := encode(t, composer.Request{V: composer.Version, Cases: []composer.Case{{Name: "c", Manifests: map[string]composer.Manifest{"x": {Digest: "abc"}}}}})
+	for _, c := range []struct {
+		name string
+		args []string
+		in   string
+		code int
+		says string
+	}{
+		{"not JSON", nil, "{", 1, "not JSON"},
+		{"another version", nil, `{"v":2}`, 1, "version 2"},
+		{"a digest left unfilled", nil, string(unfilled), 1, "digest"},
+		{"an unknown depth", nil, `{"v":1,"depth":"deep"}`, 1, "depth"},
+		{"nothing to read", nil, "", 1, "no request"},
+		{"a wrong flag", []string{"-nope"}, "", 2, "nope"},
+		{"an argument", []string{"extra"}, "", 2, "unexpected"},
+	} {
+		var out, errOut bytes.Buffer
+		if got := run(c.args, strings.NewReader(c.in), &out, &errOut); got != c.code || !strings.Contains(errOut.String(), c.says) {
+			t.Errorf("%s: exit %d saying %q, want %d saying %q", c.name, got, errOut.String(), c.code, c.says)
+		}
+	}
+}
+
+// sdkPackages is every package of this module the command imports, directly or not, with the packages each imports.
+func sdkPackages(t *testing.T) map[string][]string {
+	t.Helper()
+	_, file, _, _ := runtime.Caller(0)
+	root := filepath.Join(filepath.Dir(file), "..", "..")
+	const module = "git.novox.be/novox/mesh-sdk/go"
+	out := map[string][]string{}
+	var walk func(path string)
+	walk = func(path string) {
+		if _, done := out[path]; done {
+			return
+		}
+		rel := strings.TrimPrefix(strings.TrimPrefix(path, module), "/")
+		pkg, err := build.Default.ImportDir(filepath.Join(root, rel), 0)
+		if err != nil {
+			t.Fatalf("%s: %v", path, err)
+		}
+		out[path] = pkg.Imports
+		for _, imp := range pkg.Imports {
+			if strings.HasPrefix(imp, module+"/") {
+				walk(imp)
+			}
+		}
+	}
+	walk(composer.Package)
+	return out
+}
+
+// 1621/AC2 and AC4: the command imports, through every package of mesh-sdk it reaches, only the standard library and
+// mesh-sdk's own packages (so `go build` from mesh-sdk's module needs nothing beyond its lock), and none that reaches
+// a bus, a store, a network service or another process.
+func TestTheCommandImportsNoBusStoreNetworkOrProcess(t *testing.T) {
+	forbidden := []string{"net/http", "net/rpc", "net/smtp", "os/exec", "database/sql", "plugin", "syscall"}
+	for path, imports := range sdkPackages(t) {
+		for _, imp := range imports {
+			std := !strings.Contains(strings.SplitN(imp, "/", 2)[0], ".")
+			if !std && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-sdk/go/") {
+				t.Errorf("%s imports %s, which is neither the standard library nor mesh-sdk", path, imp)
+			}
+			for _, f := range forbidden {
+				if imp == f {
+					t.Errorf("%s imports %s", path, imp)
+				}
+			}
+		}
+	}
+}
+
+// 1621/AC2: run with an empty network namespace, the command still writes its verdict: it needs no network. The test
+// runs its own binary again under `unshare --net` as an unprivileged user's namespace; where the kernel or the sandbox
+// allows no new namespace (a build module's run step is already one with no network), it says so and skips.
+func TestTheCommandRunsWithNoNetwork(t *testing.T) {
+	if os.Getenv("MESH_COMPOSE_HELPER") == "1" {
+		os.Exit(run(nil, os.Stdin, os.Stdout, os.Stderr))
+	}
+	unshare, err := exec.LookPath("unshare")
+	if err != nil {
+		t.Skip("no unshare on this machine, so no empty network namespace can be made here")
+	}
+	if out, err := exec.Command(unshare, "--net", "--map-root-user", "true").CombinedOutput(); err != nil {
+		t.Skipf("this machine makes no unprivileged network namespace (%v: %s); a run step's own has no network", err, out)
+	}
+	r := request(t, composer.DepthResolve)
+	cmd := exec.Command(unshare, "--net", "--map-root-user", os.Args[0], "-test.run=^TestTheCommandRunsWithNoNetwork$")
+	cmd.Env = []string{"MESH_COMPOSE_HELPER=1"}
+	cmd.Stdin = bytes.NewReader(encode(t, r))
+	var out, errOut bytes.Buffer
+	cmd.Stdout, cmd.Stderr = &out, &errOut
+	if err := cmd.Run(); err != nil {
+		t.Fatalf("the command in an empty network namespace: %v: %s", err, errOut.String())
+	}
+	var v composer.Verdict
+	if err := json.Unmarshal(out.Bytes(), &v); err != nil || len(v.Cases) != 1 || len(v.Cases[0].Machines) != 3 {
+		t.Fatalf("no verdict from the command with no network: %v\n%s", err, out.String())
+	}
+	names := make([]string, 0, len(v.Cases[0].Machines))
+	for n := range v.Cases[0].Machines {
+		names = append(names, n)
+	}
+	sort.Strings(names)
+	t.Logf("composed with no network: %v", names)
+}