From 20d18eb4f5c38029f2d3e41560eb8d49e61568f6 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 24 Oct 2026 03:51:35 +0200 Subject: [PATCH 1/9] Give mesh-sdk a composer command, so a program linking one mesh-sdk can have another commit's composition judged (issue 1621) --- go/cmd/mesh-compose/main.go | 144 ++++++++++++++ go/cmd/mesh-compose/main_test.go | 293 +++++++++++++++++++++++++++ go/composer/composer.go | 332 +++++++++++++++++++++++++++++++ go/composer/composer_test.go | 106 ++++++++++ 4 files changed, 875 insertions(+) create mode 100644 go/cmd/mesh-compose/main.go create mode 100644 go/cmd/mesh-compose/main_test.go create mode 100644 go/composer/composer.go create mode 100644 go/composer/composer_test.go diff --git a/go/cmd/mesh-compose/main.go b/go/cmd/mesh-compose/main.go new file mode 100644 index 0000000..e871b0a --- /dev/null +++ b/go/cmd/mesh-compose/main.go @@ -0,0 +1,144 @@ +// mesh-compose is mesh-sdk's composer command (novox/hq ADR 0482 clause 4, issue 1621): it reads a composer.Request on +// standard input, validates every manifest with this mesh-sdk's reader, composes the machines named in each case and +// writes one composer.Verdict on standard output. Built at any commit of mesh-sdk (or at the mesh-sdk a module pins), +// it answers what that commit's composition answers, so a program that links one mesh-sdk can have another's judged. +// +// mesh-compose < request.json > verdict.json +// mesh-compose -catalogue [-node ]... [-depth resolve|compose] [-declarations] [< request.json] +// +// -catalogue reads /modules/*/module.json and a module.json at 's root as the first case's manifests (a +// checkout or `git archive` of the catalogue repository at a ref: the command fetches nothing). -node adds a machine to +// compose; -declarations answers each machine's declared paths with their owners and modes. +// +// It opens no network connection, bus, store or file beyond its standard input and the -catalogue directory, and reads +// no environment. Exit codes: 0 the verdict is written (a machine that does not compose is a verdict, not an error); +// 1 the request cannot be read or run, said on standard error; 2 the flags are wrong. +package main + +import ( + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + + "git.novox.be/novox/mesh-sdk/go/composer" +) + +func main() { + os.Exit(run(os.Args[1:], os.Stdin, os.Stdout, os.Stderr)) +} + +type nodes []string + +func (n *nodes) String() string { return strings.Join(*n, ",") } +func (n *nodes) Set(v string) error { *n = append(*n, v); return nil } + +// run is the command: its arguments, standard input, output and error; it answers the exit code. +func run(args []string, stdin io.Reader, stdout, stderr io.Writer) int { + fs := flag.NewFlagSet("mesh-compose", flag.ContinueOnError) + fs.SetOutput(stderr) + catalogueDir := fs.String("catalogue", "", "a catalogue checkout: /modules/*/module.json are the first case's manifests") + var machines nodes + fs.Var(&machines, "node", "a machine to compose (repeatable)") + depth := fs.String("depth", "", "resolve or compose (default: the request's, else compose)") + declarations := fs.Bool("declarations", false, "answer each composed machine's declared paths, owners and modes") + if err := fs.Parse(args); err != nil { + return 2 + } + if fs.NArg() > 0 { + fmt.Fprintf(stderr, "mesh-compose: unexpected arguments %q\n", fs.Args()) + return 2 + } + raw, err := io.ReadAll(stdin) + if err != nil { + fmt.Fprintln(stderr, "mesh-compose: standard input cannot be read:", err) + return 1 + } + var req composer.Request + if strings.TrimSpace(string(raw)) == "" { + if *catalogueDir == "" { + fmt.Fprintln(stderr, "mesh-compose: no request on standard input and no -catalogue") + return 1 + } + req = composer.Request{V: composer.Version} + } else if req, err = composer.Decode(raw); err != nil { + fmt.Fprintln(stderr, "mesh-compose:", err) + return 1 + } + if *catalogueDir != "" { + read, err := readCatalogue(*catalogueDir) + if err != nil { + fmt.Fprintln(stderr, "mesh-compose:", err) + return 1 + } + if len(req.Cases) == 0 { + req.Cases = []composer.Case{{Name: "catalogue", Manifests: map[string]composer.Manifest{}}} + } + if req.Cases[0].Manifests == nil { + req.Cases[0].Manifests = map[string]composer.Manifest{} + } + for name, m := range read { + if _, given := req.Cases[0].Manifests[name]; !given { + req.Cases[0].Manifests[name] = m + } + } + } + req.Machines = append(req.Machines, machines...) + if *depth != "" { + req.Depth = *depth + } + if *declarations { + req.Declarations = true + } + v, err := composer.Run(req) + if err != nil { + fmt.Fprintln(stderr, "mesh-compose:", err) + return 1 + } + enc := json.NewEncoder(stdout) + if err := enc.Encode(v); err != nil { + fmt.Fprintln(stderr, "mesh-compose: the verdict cannot be written:", err) + return 1 + } + return 0 +} + +// readCatalogue reads every module.json of a catalogue checkout by its module's name: the root's and each of +// modules/*/. A manifest whose name cannot be read is kept under its directory's name, so the reader's refusal is said. +func readCatalogue(dir string) (map[string]composer.Manifest, error) { + paths, err := filepath.Glob(filepath.Join(dir, "modules", "*", "module.json")) + if err != nil { + return nil, err + } + if _, err := os.Stat(filepath.Join(dir, "module.json")); err == nil { + paths = append(paths, filepath.Join(dir, "module.json")) + } + if len(paths) == 0 { + return nil, fmt.Errorf("the catalogue %s holds no module.json", dir) + } + sort.Strings(paths) + out := map[string]composer.Manifest{} + for _, p := range paths { + raw, err := os.ReadFile(p) + if err != nil { + return nil, err + } + var probe struct { + Module string `json:"module"` + } + name := filepath.Base(filepath.Dir(p)) + if json.Unmarshal(raw, &probe) == nil && probe.Module != "" { + name = probe.Module + } + if _, twice := out[name]; twice { + return nil, errors.New("the catalogue " + dir + " defines " + name + " twice") + } + out[name] = composer.Manifest{Raw: raw} + } + return out, nil +} diff --git a/go/cmd/mesh-compose/main_test.go b/go/cmd/mesh-compose/main_test.go new file mode 100644 index 0000000..358e743 --- /dev/null +++ b/go/cmd/mesh-compose/main_test.go @@ -0,0 +1,293 @@ +package main + +import ( + "bytes" + "encoding/json" + "go/build" + "os" + "os/exec" + "path/filepath" + "reflect" + "runtime" + "sort" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/compose" + "git.novox.be/novox/mesh-sdk/go/composer" + "git.novox.be/novox/mesh-sdk/go/internal/beside" +) + +// captured is every manifest of the catalogue captured in testdata/beside (internal/beside), raw, by module name. +func captured(t *testing.T) map[string]composer.Manifest { + t.Helper() + read, err := readCatalogue(filepath.Dir(beside.Catalogue(t))) + if err != nil { + t.Fatal(err) + } + if len(read) < 100 { + t.Fatalf("the captured catalogue holds %d manifests, fewer than the catalogue's", len(read)) + } + return read +} + +// meshOf is a mesh of three machines over the captured catalogue, each assigned real modules, the input the assigner +// composes with (nodes and their assignments, holdings from the claims): anchor runs the bus and the store, laptop a +// desktop's modules, ace a module whose requirement is provided elsewhere. +func meshOf() compose.Input { + return compose.Input{Nodes: []compose.Node{ + {Name: "anchor", Assigned: []string{"nats", "postgres"}}, + {Name: "laptop", Assigned: []string{"sudo", "zsh"}}, + {Name: "ace", Assigned: []string{"mesh-reads", "nats"}}, + }} +} + +func request(t *testing.T, depth string) composer.Request { + t.Helper() + return composer.Request{V: composer.Version, Input: meshOf(), Depth: depth, HoldingsFromClaims: true, + Machines: []string{"anchor", "laptop", "ace"}, + Cases: []composer.Case{{Name: "trunk", Manifests: captured(t)}}} +} + +func runJSON(t *testing.T, args []string, in []byte) (int, composer.Verdict, string) { + t.Helper() + var out, errOut bytes.Buffer + code := run(args, bytes.NewReader(in), &out, &errOut) + var v composer.Verdict + if code == 0 { + if err := json.Unmarshal(out.Bytes(), &v); err != nil { + t.Fatalf("the verdict is not JSON: %v\n%s", err, out.String()) + } + } + return code, v, errOut.String() +} + +func encode(t *testing.T, r composer.Request) []byte { + t.Helper() + raw, err := json.Marshal(r) + if err != nil { + t.Fatal(err) + } + return raw +} + +// 1621/AC1: the command reads the plain input and the manifests on standard input, validates every manifest, composes +// the machines named and writes one verdict: per machine composes or the reason, per manifest accepted or the field +// refused. +func TestTheCommandWritesOneVerdictPerManifestAndMachine(t *testing.T) { + r := request(t, composer.DepthResolve) + r.Cases[0].Manifests["broken"] = composer.Manifest{Raw: json.RawMessage(`{"module":"broken","version":"1","shares":["x"]}`)} + code, v, stderr := runJSON(t, nil, encode(t, r)) + if code != 0 { + t.Fatalf("exit %d: %s", code, stderr) + } + if len(v.Cases) != 1 || v.Cases[0].Name != "trunk" || v.SDK == "" { + t.Fatalf("verdict = %+v", v) + } + c := v.Cases[0] + if got := c.Manifests["broken"]; got.Accepted || !strings.Contains(got.Refused, `"shares"`) { + t.Errorf("the manifest with an unknown field reads %+v, want refused naming the field", got) + } + if got := c.Manifests["nats"]; !got.Accepted { + t.Errorf("nats reads %+v, want accepted", got) + } + if len(c.Machines) != 3 { + t.Fatalf("machines = %v, want the three named", c.Machines) + } + for name, m := range c.Machines { + if !m.Composes && m.Reason == "" { + t.Errorf("%s does not compose and says no reason", name) + } + } +} + +// 1621/AC3: over the captured catalogue and a mesh of three machines, the command's verdict equals, machine for +// machine and manifest for manifest, what the linked reader, resolution and composition answer, at both depths. +func TestTheCommandAnswersWhatTheLinkedCompositionAnswers(t *testing.T) { + for _, depth := range []string{composer.DepthResolve, composer.DepthCompose} { + r := request(t, depth) + code, v, stderr := runJSON(t, nil, encode(t, r)) + if code != 0 { + t.Fatalf("%s: exit %d: %s", depth, code, stderr) + } + in := meshOf() + in.Catalogue = map[string]catalogue.Manifest{} + for name, m := range r.Cases[0].Manifests { + parsed, err := catalogue.ParseManifest(m.Raw) + got := v.Cases[0].Manifests[name] + if (err == nil) != got.Accepted || (err != nil && err.Error() != got.Refused) { + t.Errorf("%s: the manifest of %s: linked %v, command %+v", depth, name, err, got) + } + if err == nil { + in.Catalogue[name] = parsed + } + } + in.Holdings = composer.HoldingsOf(in.Catalogue, in.Nodes) + composed := 0 + for _, node := range r.Machines { + got := v.Cases[0].Machines[node] + var want composer.Machine + if depth == composer.DepthResolve { + if _, err := compose.Resolve(in, node); err != nil { + want = composer.Machine{Reason: err.Error()} + } else { + want = composer.Machine{Composes: true} + } + } else { + d, problems, err := compose.Compose(in, node) + switch { + case err != nil: + want = composer.Machine{Reason: err.Error()} + default: + want = composer.Machine{Composes: len(problems) == 0, Problems: problems, Refusals: d.Refusals()} + if len(problems) > 0 { + want.Reason = "the node-engine's validator refuses the declaration" + } + } + } + if !reflect.DeepEqual(got, want) { + t.Errorf("%s: %s: command %+v, linked %+v", depth, node, got, want) + } + if got.Composes { + composed++ + } else { + t.Logf("%s: %s does not compose: %s %v", depth, node, got.Reason, got.Problems) + } + } + t.Logf("%s: %d of %d machines compose", depth, composed, len(r.Machines)) + } +} + +// 1621 note 6: a person composes a catalogue checkout (repository@ref checked out or archived: the command fetches +// nothing) for a node and reads its declaration: each path with its kind, owner and mode. +func TestACatalogueCheckoutIsComposedForANode(t *testing.T) { + dir := t.TempDir() + if err := os.MkdirAll(filepath.Join(dir, "modules", "demo"), 0o755); err != nil { + t.Fatal(err) + } + manifest := `{"module": "demo", "version": "1", "resources": [{"id": "state", "type": "directory", ` + + `"path": "/var/lib/demo", "mode": "0750", "owner": "root"}]}` + if err := os.WriteFile(filepath.Join(dir, "modules", "demo", "module.json"), []byte(manifest), 0o644); err != nil { + t.Fatal(err) + } + in := encode(t, composer.Request{V: composer.Version, Input: compose.Input{Nodes: []compose.Node{{Name: "laptop", Assigned: []string{"demo"}}}}}) + code, v, stderr := runJSON(t, []string{"-catalogue", dir, "-node", "laptop", "-declarations"}, in) + if code != 0 { + t.Fatalf("exit %d: %s", code, stderr) + } + m := v.Cases[0].Machines["laptop"] + want := []composer.Path{{Path: "/var/lib/demo", Kind: "directory", Owner: "root", Mode: "0750", Module: ""}} + if !m.Composes || len(m.Declared) != 1 || m.Declared[0].Path != want[0].Path || m.Declared[0].Mode != "0750" || + m.Declared[0].Owner != "root" || m.Declared[0].Kind != "directory" { + t.Fatalf("laptop = %+v, want it composed declaring %+v", m, want) + } +} + +// The exit codes: 1 for a request that cannot be read or run, 2 for wrong flags, each said on standard error. +func TestTheExitCodesSayWhatFailed(t *testing.T) { + unfilled := encode(t, composer.Request{V: composer.Version, Cases: []composer.Case{{Name: "c", Manifests: map[string]composer.Manifest{"x": {Digest: "abc"}}}}}) + for _, c := range []struct { + name string + args []string + in string + code int + says string + }{ + {"not JSON", nil, "{", 1, "not JSON"}, + {"another version", nil, `{"v":2}`, 1, "version 2"}, + {"a digest left unfilled", nil, string(unfilled), 1, "digest"}, + {"an unknown depth", nil, `{"v":1,"depth":"deep"}`, 1, "depth"}, + {"nothing to read", nil, "", 1, "no request"}, + {"a wrong flag", []string{"-nope"}, "", 2, "nope"}, + {"an argument", []string{"extra"}, "", 2, "unexpected"}, + } { + var out, errOut bytes.Buffer + if got := run(c.args, strings.NewReader(c.in), &out, &errOut); got != c.code || !strings.Contains(errOut.String(), c.says) { + t.Errorf("%s: exit %d saying %q, want %d saying %q", c.name, got, errOut.String(), c.code, c.says) + } + } +} + +// sdkPackages is every package of this module the command imports, directly or not, with the packages each imports. +func sdkPackages(t *testing.T) map[string][]string { + t.Helper() + _, file, _, _ := runtime.Caller(0) + root := filepath.Join(filepath.Dir(file), "..", "..") + const module = "git.novox.be/novox/mesh-sdk/go" + out := map[string][]string{} + var walk func(path string) + walk = func(path string) { + if _, done := out[path]; done { + return + } + rel := strings.TrimPrefix(strings.TrimPrefix(path, module), "/") + pkg, err := build.Default.ImportDir(filepath.Join(root, rel), 0) + if err != nil { + t.Fatalf("%s: %v", path, err) + } + out[path] = pkg.Imports + for _, imp := range pkg.Imports { + if strings.HasPrefix(imp, module+"/") { + walk(imp) + } + } + } + walk(composer.Package) + return out +} + +// 1621/AC2 and AC4: the command imports, through every package of mesh-sdk it reaches, only the standard library and +// mesh-sdk's own packages (so `go build` from mesh-sdk's module needs nothing beyond its lock), and none that reaches +// a bus, a store, a network service or another process. +func TestTheCommandImportsNoBusStoreNetworkOrProcess(t *testing.T) { + forbidden := []string{"net/http", "net/rpc", "net/smtp", "os/exec", "database/sql", "plugin", "syscall"} + for path, imports := range sdkPackages(t) { + for _, imp := range imports { + std := !strings.Contains(strings.SplitN(imp, "/", 2)[0], ".") + if !std && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-sdk/go/") { + t.Errorf("%s imports %s, which is neither the standard library nor mesh-sdk", path, imp) + } + for _, f := range forbidden { + if imp == f { + t.Errorf("%s imports %s", path, imp) + } + } + } + } +} + +// 1621/AC2: run with an empty network namespace, the command still writes its verdict: it needs no network. The test +// runs its own binary again under `unshare --net` as an unprivileged user's namespace; where the kernel or the sandbox +// allows no new namespace (a build module's run step is already one with no network), it says so and skips. +func TestTheCommandRunsWithNoNetwork(t *testing.T) { + if os.Getenv("MESH_COMPOSE_HELPER") == "1" { + os.Exit(run(nil, os.Stdin, os.Stdout, os.Stderr)) + } + unshare, err := exec.LookPath("unshare") + if err != nil { + t.Skip("no unshare on this machine, so no empty network namespace can be made here") + } + if out, err := exec.Command(unshare, "--net", "--map-root-user", "true").CombinedOutput(); err != nil { + t.Skipf("this machine makes no unprivileged network namespace (%v: %s); a run step's own has no network", err, out) + } + r := request(t, composer.DepthResolve) + cmd := exec.Command(unshare, "--net", "--map-root-user", os.Args[0], "-test.run=^TestTheCommandRunsWithNoNetwork$") + cmd.Env = []string{"MESH_COMPOSE_HELPER=1"} + cmd.Stdin = bytes.NewReader(encode(t, r)) + var out, errOut bytes.Buffer + cmd.Stdout, cmd.Stderr = &out, &errOut + if err := cmd.Run(); err != nil { + t.Fatalf("the command in an empty network namespace: %v: %s", err, errOut.String()) + } + var v composer.Verdict + if err := json.Unmarshal(out.Bytes(), &v); err != nil || len(v.Cases) != 1 || len(v.Cases[0].Machines) != 3 { + t.Fatalf("no verdict from the command with no network: %v\n%s", err, out.String()) + } + names := make([]string, 0, len(v.Cases[0].Machines)) + for n := range v.Cases[0].Machines { + names = append(names, n) + } + sort.Strings(names) + t.Logf("composed with no network: %v", names) +} diff --git a/go/composer/composer.go b/go/composer/composer.go new file mode 100644 index 0000000..5ab962a --- /dev/null +++ b/go/composer/composer.go @@ -0,0 +1,332 @@ +// Package composer is mesh-sdk's composition and validator as one function over a plain request (novox/hq ADR 0482 +// clause 4, issue 1621): what the command mesh-compose runs, so a program that links any one mesh-sdk can have the +// composition of another commit judged by building that commit's command and reading its verdict. +// +// **It reads its request and nothing else.** No bus, store, network, file or environment: the manifests come inside the +// request (raw, as their repository holds them), the rest of the mesh is a compose.Input, and the answer is a value. +// A manifest named only by its digest in the delivery indexer's bucket is filled before the run by Fill, by whoever +// holds the bus (the Go build module's compose job); Run refuses a digest left unfilled. +package composer + +import ( + "encoding/json" + "errors" + "fmt" + "slices" + "sort" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/compose" + "git.novox.be/novox/mesh-sdk/go/sdkversion" +) + +// Version is the version of a Request and of a Verdict; a reader refuses another. +const Version = 1 + +// Package is the import path of the command, which a compose job builds at a commit (in mesh-sdk's own module, or in +// a module that requires mesh-sdk, at the version it pins). +const Package = "git.novox.be/novox/mesh-sdk/go/cmd/mesh-compose" + +// The depths of a composition. +const ( + // DepthResolve resolves each machine's set over the rest of the mesh (compose.Resolve): what the assigner judges + // while the facts snapshot is not on the bus (issue 1461). + DepthResolve = "resolve" + // DepthCompose composes each machine's whole declaration and runs the node-engine's validator over it + // (compose.Compose). + DepthCompose = "compose" +) + +// Request is everything one run composes: the rest of the mesh, one or more cases of manifests composed over it, and +// the machines composed in each case. +type Request struct { + V int `json:"v"` + // Input is the composition's input without its catalogue (and without each node's kept manifests): the cases give + // those. A Catalogue or Kept it holds is ignored. + Input compose.Input `json:"input"` + // Cases are composed one after the other over Input, each with its own manifests; their names are unique. + Cases []Case `json:"cases"` + // Machines are the nodes composed in every case; none means the manifests are only read. + Machines []string `json:"machines,omitempty"` + // Depth is DepthResolve or DepthCompose; empty is DepthCompose. + Depth string `json:"depth,omitempty"` + // HoldingsFromClaims adds to Input's holdings every seat an assigned module claims, held on its node (the + // assigner's input, which has no holdings of record). + HoldingsFromClaims bool `json:"holdings_from_claims,omitempty"` + // Declarations asks each composed machine's declared paths (DepthCompose only). + Declarations bool `json:"declarations,omitempty"` +} + +// Case is one catalogue composed: every module's manifest by name, and per node the manifests kept at the build it runs. +type Case struct { + Name string `json:"name"` + Manifests map[string]Manifest `json:"manifests"` + Kept map[string]map[string]Manifest `json:"kept,omitempty"` +} + +// Manifest is a module's manifest as its repository holds it (Raw, the bytes), or the digest the delivery indexer keeps +// it under (Digest), which Fill replaces by the bytes. On the wire it is the manifest itself, or {"digest": "..."}. +type Manifest struct { + Raw json.RawMessage + Digest string +} + +// MarshalJSON writes the raw manifest, or {"digest": ...} when only the digest is known. +func (m Manifest) MarshalJSON() ([]byte, error) { + if len(m.Raw) > 0 { + return m.Raw, nil + } + return json.Marshal(map[string]string{"digest": m.Digest}) +} + +// UnmarshalJSON reads {"digest": "..."} (an object with that one key) as a digest, and any other value as the raw +// manifest. +func (m *Manifest) UnmarshalJSON(raw []byte) error { + var probe map[string]json.RawMessage + if json.Unmarshal(raw, &probe) == nil && len(probe) == 1 { + if d, ok := probe["digest"]; ok { + var digest string + if err := json.Unmarshal(d, &digest); err != nil { + return fmt.Errorf("a manifest's digest is not a string: %w", err) + } + *m = Manifest{Digest: digest} + return nil + } + } + *m = Manifest{Raw: slices.Clone(raw)} + return nil +} + +// Verdict is what one run answers: the mesh-sdk the command was built with, and each case's verdict in the order asked. +type Verdict struct { + V int `json:"v"` + SDK string `json:"sdk"` + Cases []CaseVerdict `json:"cases"` +} + +// CaseVerdict is one case's: every manifest read or refused, every machine composed or not. +type CaseVerdict struct { + Name string `json:"name"` + Manifests map[string]ManifestRead `json:"manifests"` + Machines map[string]Machine `json:"machines,omitempty"` +} + +// ManifestRead is one manifest as this mesh-sdk reads it: accepted, or refused with the reader's words. +type ManifestRead struct { + Accepted bool `json:"accepted"` + Refused string `json:"refused,omitempty"` +} + +// Machine is one machine composed: whether it composes (resolves, at DepthResolve), why not, the node-engine +// validator's problems and the composition's refusals (DepthCompose), and its declared paths when asked. +type Machine struct { + Composes bool `json:"composes"` + Reason string `json:"reason,omitempty"` + Problems []string `json:"problems,omitempty"` + Refusals []string `json:"refusals,omitempty"` + Declared []Path `json:"declared,omitempty"` +} + +// Path is one path a declaration puts on the machine, with its kind (the resource's type), owner, group and mode as +// declared, and the module it is of. +type Path struct { + Path string `json:"path"` + Kind string `json:"kind"` + Owner string `json:"owner,omitempty"` + Group string `json:"group,omitempty"` + Mode string `json:"mode,omitempty"` + Module string `json:"module,omitempty"` +} + +// ErrUnfilled is a manifest named by a digest that nobody replaced by its bytes. +var ErrUnfilled = errors.New("composer: a manifest is named by its digest and was not filled") + +// Decode reads a request; it refuses one that is not JSON or of another version. +func Decode(data []byte) (Request, error) { + var r Request + if err := json.Unmarshal(data, &r); err != nil { + return Request{}, fmt.Errorf("composer: the request is not JSON of its shape: %w", err) + } + if r.V != Version { + return Request{}, fmt.Errorf("composer: a request of version %d, this composer reads %d", r.V, Version) + } + return r, nil +} + +// Fill replaces every manifest named by its digest by the bytes get answers for that digest; a digest get cannot answer +// stops it, named. +func Fill(r *Request, get func(digest string) ([]byte, error)) error { + fill := func(m map[string]Manifest) error { + for name, man := range m { + if len(man.Raw) > 0 { + continue + } + raw, err := get(man.Digest) + if err != nil { + return fmt.Errorf("composer: the manifest of %s (digest %s): %w", name, man.Digest, err) + } + m[name] = Manifest{Raw: raw} + } + return nil + } + for _, c := range r.Cases { + if err := fill(c.Manifests); err != nil { + return err + } + for _, kept := range c.Kept { + if err := fill(kept); err != nil { + return err + } + } + } + return nil +} + +// Run composes every case of r and answers the verdict, its SDK this binary's mesh-sdk. An error is a request it +// cannot run (an unknown depth, a case named twice, a manifest left as a digest); a machine that does not compose is a +// verdict. +func Run(r Request) (Verdict, error) { + depth := r.Depth + if depth == "" { + depth = DepthCompose + } + if depth != DepthResolve && depth != DepthCompose { + return Verdict{}, fmt.Errorf("composer: the depth %q is not resolve or compose", r.Depth) + } + v := Verdict{V: Version, SDK: sdkversion.Version()} + seen := map[string]bool{} + for _, c := range r.Cases { + if seen[c.Name] { + return Verdict{}, fmt.Errorf("composer: the case %q is named twice", c.Name) + } + seen[c.Name] = true + cv, err := runCase(r, c, depth) + if err != nil { + return Verdict{}, err + } + v.Cases = append(v.Cases, cv) + } + return v, nil +} + +func parse(m map[string]Manifest, reads map[string]ManifestRead) (map[string]catalogue.Manifest, error) { + out := map[string]catalogue.Manifest{} + for name, man := range m { + if len(man.Raw) == 0 { + return nil, fmt.Errorf("%w: %s (digest %s)", ErrUnfilled, name, man.Digest) + } + parsed, err := catalogue.ParseManifest(man.Raw) + if reads != nil { + if err != nil { + reads[name] = ManifestRead{Refused: err.Error()} + } else { + reads[name] = ManifestRead{Accepted: true} + } + } + if err == nil { + out[name] = parsed + } + } + return out, nil +} + +func runCase(r Request, c Case, depth string) (CaseVerdict, error) { + cv := CaseVerdict{Name: c.Name, Manifests: map[string]ManifestRead{}} + shelf, err := parse(c.Manifests, cv.Manifests) + if err != nil { + return CaseVerdict{}, err + } + in := r.Input + in.Catalogue = shelf + in.Nodes = slices.Clone(in.Nodes) + in.Holdings = slices.Clone(in.Holdings) + for i := range in.Nodes { + in.Nodes[i].Kept = nil + if kept := c.Kept[in.Nodes[i].Name]; len(kept) > 0 { + k, err := parse(kept, nil) + if err != nil { + return CaseVerdict{}, err + } + in.Nodes[i].Kept = k + } + } + if r.HoldingsFromClaims { + in.Holdings = append(in.Holdings, HoldingsOf(shelf, in.Nodes)...) + } + if len(r.Machines) == 0 { + return cv, nil + } + cv.Machines = map[string]Machine{} + for _, node := range r.Machines { + cv.Machines[node] = machine(in, node, depth, r.Declarations) + } + return cv, nil +} + +// HoldingsOf is every seat an assigned module claims, held on the node it is assigned to, in node order: the holdings +// of an input that has none of record (the assigner's, mesh-delivery-assigner internal/assign input). +func HoldingsOf(shelf map[string]catalogue.Manifest, nodes []compose.Node) []catalogue.Held { + var out []catalogue.Held + sorted := slices.Clone(nodes) + sort.Slice(sorted, func(i, j int) bool { return sorted[i].Name < sorted[j].Name }) + for _, n := range sorted { + for _, mod := range n.Assigned { + for _, cl := range shelf[mod].Claims { + scope := cl.Scope + if seat, ok := catalogue.SeatNamed(cl.Name); ok && scope == "" { + scope = seat.Scope + } + out = append(out, catalogue.Held{Claim: cl.Name, Scope: scope, Node: n.Name, Module: mod}) + } + } + } + return out +} + +func machine(in compose.Input, node, depth string, declarations bool) Machine { + if depth == DepthResolve { + if _, err := compose.Resolve(in, node); err != nil { + return Machine{Reason: err.Error()} + } + return Machine{Composes: true} + } + d, problems, err := compose.Compose(in, node) + if err != nil { + return Machine{Reason: err.Error()} + } + m := Machine{Composes: len(problems) == 0, Problems: problems, Refusals: d.Refusals()} + if len(problems) > 0 { + m.Reason = "the node-engine's validator refuses the declaration" + } + if declarations { + m.Declared = pathsOf(d) + } + return m +} + +// pathsOf is every resource of a declaration that names a path, with its owner, group and mode as declared, sorted by +// path. +func pathsOf(d compose.Declared) []Path { + var out []Path + str := func(r map[string]any, key string) string { + if v, ok := r[key]; ok && v != nil { + return fmt.Sprint(v) + } + return "" + } + for _, r := range d.Resources { + p := str(r, "path") + if p == "" { + continue + } + out = append(out, Path{Path: p, Kind: str(r, "type"), Owner: str(r, "owner"), Group: str(r, "group"), + Mode: str(r, "mode"), Module: d.ModuleOf[str(r, "id")]}) + } + sort.Slice(out, func(i, j int) bool { + if out[i].Path != out[j].Path { + return out[i].Path < out[j].Path + } + return out[i].Kind < out[j].Kind + }) + return out +} diff --git a/go/composer/composer_test.go b/go/composer/composer_test.go new file mode 100644 index 0000000..175894a --- /dev/null +++ b/go/composer/composer_test.go @@ -0,0 +1,106 @@ +package composer + +import ( + "encoding/json" + "errors" + "reflect" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/compose" +) + +const demo = `{"module": "demo", "version": "1", "claims": [{"name": "node-files"}]}` + +// A manifest named by its digest goes on the wire as {"digest": ...}, and Fill puts the bytes the indexer keeps in its +// place; a raw manifest goes as itself. +func TestAManifestIsItsBytesOrItsDigest(t *testing.T) { + r := Request{V: Version, Cases: []Case{{Name: "change", Manifests: map[string]Manifest{ + "demo": {Digest: "d1"}, "raw": {Raw: json.RawMessage(`{"module":"raw","version":"1"}`)}}, + Kept: map[string]map[string]Manifest{"laptop": {"demo": {Digest: "d1"}}}}}} + wire, err := json.Marshal(r) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(wire), `"demo":{"digest":"d1"}`) || !strings.Contains(string(wire), `"raw":{"module":"raw","version":"1"}`) { + t.Fatalf("on the wire: %s", wire) + } + back, err := Decode(wire) + if err != nil { + t.Fatal(err) + } + if _, err := Run(back); !errors.Is(err, ErrUnfilled) { + t.Fatalf("a digest left unfilled runs: %v", err) + } + asked := 0 + if err := Fill(&back, func(d string) ([]byte, error) { + asked++ + if d != "d1" { + return nil, errors.New("no such digest") + } + return []byte(demo), nil + }); err != nil { + t.Fatal(err) + } + if asked != 2 || string(back.Cases[0].Manifests["demo"].Raw) != demo || string(back.Cases[0].Kept["laptop"]["demo"].Raw) != demo { + t.Fatalf("filled %d times: %+v", asked, back.Cases[0]) + } + bad := Request{V: Version, Cases: []Case{{Name: "c", Manifests: map[string]Manifest{"x": {Digest: "nope"}}}}} + if err := Fill(&bad, func(string) ([]byte, error) { return nil, errors.New("not there") }); err == nil || !strings.Contains(err.Error(), "nope") { + t.Fatalf("a digest the indexer does not hold: %v", err) + } +} + +// The input travels as JSON and comes back equal, so the command composes what its caller meant. +func TestTheInputTravelsAsJSON(t *testing.T) { + in := compose.Input{ + Nodes: []compose.Node{{Name: "anchor", Assigned: []string{"nats"}, Capabilities: map[string]bool{"systemd": true}, + Place: compose.Place{Endpoint: "anchor.example:51820", Site: "here", Hub: true, Address: "10.77.0.1"}, + Pins: map[string]catalogue.Chosen{"bus": {Node: "anchor", Module: "nats"}}, + Settings: map[string][]catalogue.Layer{"nats": {{Values: map[string]any{"port": float64(4222)}}}}}}, + Holdings: []catalogue.Held{{Claim: "mesh-broker", Scope: "mesh", Node: "anchor", Module: "nats"}}, + Issued: []compose.Issued{{Provision: "bus", Consumer: "laptop", ConsumerModule: "x", Provider: "anchor"}}, + } + raw, err := json.Marshal(Request{V: Version, Input: in}) + if err != nil { + t.Fatal(err) + } + back, err := Decode(raw) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(back.Input, in) { + t.Fatalf("the input came back as\n%+v\nwant\n%+v", back.Input, in) + } +} + +// The seats a node's assigned modules claim are held there when the request says the holdings come from the claims. +func TestHoldingsComeFromTheClaims(t *testing.T) { + m, err := catalogue.ParseManifest([]byte(demo)) + if err != nil { + t.Fatal(err) + } + got := HoldingsOf(map[string]catalogue.Manifest{"demo": m}, []compose.Node{{Name: "b", Assigned: []string{"demo"}}, {Name: "a", Assigned: []string{"demo"}}}) + if len(got) != 2 || got[0].Node != "a" || got[0].Claim != "node-files" || got[0].Scope != "node" { + t.Fatalf("holdings = %+v", got) + } +} + +// A request the command cannot run is refused, never answered as a verdict: another version, a case named twice, an +// unknown depth. +func TestARequestItCannotRunIsRefused(t *testing.T) { + if _, err := Decode([]byte(`{"v": 2}`)); err == nil { + t.Error("a request of version 2 is read") + } + if _, err := Run(Request{V: Version, Cases: []Case{{Name: "a"}, {Name: "a"}}}); err == nil { + t.Error("a case named twice runs") + } + if _, err := Run(Request{V: Version, Depth: "deep"}); err == nil { + t.Error("an unknown depth runs") + } + v, err := Run(Request{V: Version, Cases: []Case{{Name: "only-read", Manifests: map[string]Manifest{"demo": {Raw: json.RawMessage(demo)}}}}}) + if err != nil || len(v.Cases) != 1 || !v.Cases[0].Manifests["demo"].Accepted || v.Cases[0].Machines != nil || v.SDK == "" { + t.Fatalf("a request with no machines: %+v, %v", v, err) + } +} From 668fc2a722e02072555626c484135746a1574248 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 24 Oct 2026 03:51:35 +0200 Subject: [PATCH 2/9] Let the Go build seat carry a compose ask and its answer, so the assigner composes a group with its mesh-sdk head (issue 1622) --- go/delivery/stepasks.go | 84 +++++++++++++++++++++++++--- go/delivery/stepasks_compose_test.go | 76 +++++++++++++++++++++++++ 2 files changed, 152 insertions(+), 8 deletions(-) create mode 100644 go/delivery/stepasks_compose_test.go diff --git a/go/delivery/stepasks.go b/go/delivery/stepasks.go index 1e8cdd3..134dae8 100644 --- a/go/delivery/stepasks.go +++ b/go/delivery/stepasks.go @@ -1,8 +1,11 @@ package delivery import ( + "crypto/sha256" + "encoding/hex" "encoding/json" "fmt" + "path" "regexp" "sort" "strings" @@ -154,8 +157,8 @@ func checkAskID(id string) error { } // StepAsk is the message on a build seat's stream: who asked what, of which kind and language, and the one whole job. -// Exactly one of Build and Check is set: a delivery ask is a build; a check ask a build (a candidate's fingerprint) or a -// check; a recheck ask a check; an image ask is always a build. +// Exactly one of Build, Check and Compose is set: a delivery ask is a build; a check ask a build (a candidate's +// fingerprint), a check or, on the Go seat, a compose; a recheck ask a check; an image ask is always a build. type StepAsk struct { V int `json:"v"` ID string `json:"id"` @@ -163,10 +166,48 @@ type StepAsk struct { Language string `json:"language"` // Asker is the module that put the ask (mesh-delivery-builder or mesh-delivery-checker); the stream's grant is what // makes it true, this field only names it to a reader. - Asker string `json:"asker"` - At time.Time `json:"at"` - Build *BuildJob `json:"build,omitempty"` - Check *CheckJob `json:"check,omitempty"` + Asker string `json:"asker"` + At time.Time `json:"at"` + Build *BuildJob `json:"build,omitempty"` + Check *CheckJob `json:"check,omitempty"` + Compose *ComposeJob `json:"compose,omitempty"` +} + +// ComposeJob is a compose ask's job (novox/hq ADR 0482 clause 4, issue 1622): fetch the commit, build the composer +// command (Package) in the Go module at Dir of the fetched tree, once per ComposerKey, run it in a run step shut off +// with Request on its standard input, and answer its standard output whole (ComposeAnswer). The assigner asks it, to +// compose a group with its mesh-sdk member's head, and to read the group's manifests with the mesh-sdk the running +// controller and node-engine pin (ADR 0482 clause 5, issue 1623). +type ComposeJob struct { + // Fetch is the fetch tool's arguments: the repository and the full commit whose mesh-sdk composes (mesh-sdk itself + // at a head, or a module that requires mesh-sdk, at the version it pins). + Fetch map[string]any `json:"fetch"` + // Dir is the directory of the Go module the composer is built in, relative to the repository's root ("go" for + // mesh-sdk, RootPath for a module at the root); Package is the composer's import path (mesh-sdk composer.Package). + Dir string `json:"dir"` + Package string `json:"package"` + // Request is the composer's request (mesh-sdk composer.Request), its manifests named by their digests in the + // indexer's ManifestsBucket: the build module fills them before the run (composer.Fill). + Request json.RawMessage `json:"request"` + // Reader names, for a reader of the ask, whose composition this is: the mesh-sdk head of a group, or a module whose + // mesh-sdk reads the group's manifests (mesh-controller, mesh-host). + Reader string `json:"reader,omitempty"` +} + +// ComposeAnswer is a compose ask's step result: whether this run built the composer or took the one kept under its +// key, the key, and the composer's verdict (mesh-sdk composer.Verdict) whole. +type ComposeAnswer struct { + Built bool `json:"built"` + Key string `json:"key"` + Verdict json.RawMessage `json:"verdict"` +} + +// ComposerKey is the key a build module keeps a built composer under: the repository, commit, directory and package it +// was built from and the toolchain it was built with, hashed (`composer-` and 16 hex digits), so each head is built once +// on a node (ADR 0482 clause 4). +func ComposerKey(repository, commit, dir, pkg string, toolchain ToolchainUsed) string { + sum := sha256.Sum256([]byte(strings.Join([]string{repository, commit, dir, pkg, toolchain.Version, toolchain.Recipe}, "\n"))) + return "composer-" + hex.EncodeToString(sum[:8]) } // BuildJob is a build ask's job: fetch the commit (unless every artifact is a copy), then build or copy each artifact, @@ -232,9 +273,17 @@ func (a StepAsk) check() error { if a.At.IsZero() { return fmt.Errorf("delivery: the ask %s names no time", a.ID) } + jobs := 0 + for _, set := range []bool{a.Build != nil, a.Check != nil, a.Compose != nil} { + if set { + jobs++ + } + } switch { - case (a.Build == nil) == (a.Check == nil): - return fmt.Errorf("delivery: the ask %s holds a build or a check, exactly one", a.ID) + case jobs != 1: + return fmt.Errorf("delivery: the ask %s holds a build, a check or a compose, exactly one", a.ID) + case a.Compose != nil && (a.Kind != AskCheck || a.Language != AskGo): + return fmt.Errorf("delivery: the ask %s is a compose, which the Go seat's check stream alone carries", a.ID) case a.Build != nil && a.Kind == AskRecheck: return fmt.Errorf("delivery: the ask %s is a build on the recheck stream, which carries checks alone", a.ID) case a.Check != nil && a.Kind == AskDelivery: @@ -245,9 +294,28 @@ func (a StepAsk) check() error { if a.Build != nil { return a.Build.check(a.ID) } + if a.Compose != nil { + return a.Compose.check(a.ID) + } return a.Check.check(a.ID) } +func (c ComposeJob) check(id string) error { + if err := checkFetchArgs(id, c.Fetch); err != nil { + return err + } + if c.Dir == "" || c.Dir != path.Clean(c.Dir) || strings.HasPrefix(c.Dir, "/") || c.Dir == ".." || strings.HasPrefix(c.Dir, "../") { + return fmt.Errorf("delivery: the compose ask %s names no clean directory inside the repository (%q)", id, c.Dir) + } + if c.Package == "" { + return fmt.Errorf("delivery: the compose ask %s names no package to build", id) + } + if !json.Valid(c.Request) || len(c.Request) == 0 || c.Request[0] != '{' { + return fmt.Errorf("delivery: the compose ask %s holds no request object", id) + } + return nil +} + func (b BuildJob) check(id string) error { if err := checkName("module", b.Module); err != nil { return err diff --git a/go/delivery/stepasks_compose_test.go b/go/delivery/stepasks_compose_test.go new file mode 100644 index 0000000..a9547f0 --- /dev/null +++ b/go/delivery/stepasks_compose_test.go @@ -0,0 +1,76 @@ +package delivery + +import ( + "encoding/json" + "reflect" + "strings" + "testing" + "time" +) + +func composeAsk() StepAsk { + return StepAsk{ID: "mh3k2x9qa1b2c5", Kind: AskCheck, Language: AskGo, Asker: "mesh-delivery-assigner", + At: time.Date(2026, 10, 24, 3, 0, 0, 0, time.UTC), + Compose: &ComposeJob{Fetch: map[string]any{"repository": "novox/mesh-sdk", "commit": askCommit}, + Dir: "go", Package: "git.novox.be/novox/mesh-sdk/go/cmd/mesh-compose", + Request: json.RawMessage(`{"v":1,"cases":[{"name":"change","manifests":{"demo":{"digest":"d1"}}}]}`), Reader: "mesh-sdk head"}} +} + +// 1622: a compose ask encodes and decodes with every field. +func TestAComposeAskRoundTrips(t *testing.T) { + a := composeAsk() + raw, err := EncodeStepAsk(a) + if err != nil { + t.Fatal(err) + } + got, err := DecodeStepAsk(raw) + if err != nil { + t.Fatal(err) + } + a.V = StepAskVersion + if !reflect.DeepEqual(got, a) { + t.Fatalf("round trip:\n got %+v\nwant %+v", got, a) + } +} + +// 1622: a compose ask goes only on the Go seat's check stream, holds no other job beside it, fetches a full commit, +// builds a package in a clean directory of the repository and carries a request object. +func TestAComposeAskIsRefusedWhereItDoesNotBelong(t *testing.T) { + for name, change := range map[string]func(*StepAsk){ + "on the delivery stream": func(a *StepAsk) { a.Kind = AskDelivery }, + "on the recheck stream": func(a *StepAsk) { a.Kind = AskRecheck }, + "on the Node seat": func(a *StepAsk) { a.Language = AskNode }, + "beside a check": func(a *StepAsk) { a.Check = checkAsk().Check }, + "with a short commit": func(a *StepAsk) { a.Compose.Fetch["commit"] = "0123456" }, + "in no directory": func(a *StepAsk) { a.Compose.Dir = "" }, + "outside the repository": func(a *StepAsk) { a.Compose.Dir = "../x" }, + "with no package": func(a *StepAsk) { a.Compose.Package = "" }, + "with no request": func(a *StepAsk) { a.Compose.Request = nil }, + "with a request not JSON": func(a *StepAsk) { a.Compose.Request = json.RawMessage(`[1]`) }, + } { + a := composeAsk() + change(&a) + if _, err := EncodeStepAsk(a); err == nil { + t.Errorf("a compose ask %s is encoded", name) + } + } +} + +// 1622/AC3: a composer is kept under one key per repository, commit, directory, package and toolchain, so two asks of +// one head build it once on a node, and another head or toolchain builds its own. +func TestAComposerIsKeptByWhatItWasBuiltFrom(t *testing.T) { + tc := ToolchainUsed{Version: "go1.26.0", Recipe: "3"} + k := ComposerKey("novox/mesh-sdk", askCommit, "go", "p", tc) + if k != ComposerKey("novox/mesh-sdk", askCommit, "go", "p", tc) || !strings.HasPrefix(k, "composer-") || len(k) != len("composer-")+16 { + t.Fatalf("key = %q", k) + } + for _, other := range []string{ + ComposerKey("novox/mesh-sdk", strings.Repeat("1", 40), "go", "p", tc), + ComposerKey("novox/mesh-controller", askCommit, ".", "p", tc), + ComposerKey("novox/mesh-sdk", askCommit, "go", "p", ToolchainUsed{Version: "go1.26.1", Recipe: "3"}), + } { + if other == k { + t.Errorf("two composers built from different inputs share the key %s", k) + } + } +} From 4784762a70ceb47be1f60623720bff0766fe9b63 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 24 Oct 2026 03:51:35 +0200 Subject: [PATCH 3/9] Name every repository's siblings in beside's comment, mesh-controller beside mesh-host's tests, as the checker places them (issue 1295) --- go/internal/beside/beside.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/go/internal/beside/beside.go b/go/internal/beside/beside.go index 93a0d40..d31ec1d 100644 --- a/go/internal/beside/beside.go +++ b/go/internal/beside/beside.go @@ -15,7 +15,12 @@ // `mesh/repo-check` runs; a repository missing there fails the test, never skips it. Which clones a // check gets is chosen from the inventory: mesh-catalog by the source of the `nats` module, mesh-host // by the source of `mesh-host`, and mesh-sdk as the controller's sibling at the commit the controller's -// go.mod pins. In a mesh where either module has no source repository nothing is +// go.mod pins. Each repository's tests read these siblings, and the checker places exactly these +// (mesh-delivery's checker, besideRules, follows this list): mesh-controller's read mesh-catalog, +// mesh-host and mesh-sdk; mesh-host's read mesh-catalog, mesh-sdk and mesh-controller (its step-nine +// test reads the controller's own manifest, novox/hq issue 1295); mesh-sdk's read mesh-catalog and +// mesh-host; mesh-catalog's read mesh-sdk and mesh-controller; mesh-delivery's read mesh-sdk and +// mesh-catalog. In a mesh where either module has no source repository nothing is // cloned, and these tests fail loudly with "not beside this check": a cause in the setup, not in the // change. And in a delivery group that holds a mesh-catalog pull request, these tests read the // catalogue's main, not the group's head. From 4e0036cc95208405c297f1ffb8a7b0d2cac91b46 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 24 Oct 2026 04:00:48 +0200 Subject: [PATCH 4/9] Mark the packages whose tests read files outside their directory, so mesh-build-go never keeps their verdict (issue 1935) --- go/catalogue/reads-outside | 1 + 1 file changed, 1 insertion(+) create mode 100644 go/catalogue/reads-outside diff --git a/go/catalogue/reads-outside b/go/catalogue/reads-outside new file mode 100644 index 0000000..976b0f5 --- /dev/null +++ b/go/catalogue/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the repository's root files or other directories): mesh-build-go never keeps this package's verdict (issue 1935). From 66bf26c0d721c0b759575d11d7c40c4a8be4c965 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 24 Oct 2026 04:10:31 +0200 Subject: [PATCH 5/9] Let the composer read its request from a file, since a run step hands it no standard input (issue 1622) --- go/cmd/mesh-compose/main.go | 15 ++++++++++++--- go/cmd/mesh-compose/main_test.go | 18 ++++++++++++++++++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/go/cmd/mesh-compose/main.go b/go/cmd/mesh-compose/main.go index e871b0a..cc0211d 100644 --- a/go/cmd/mesh-compose/main.go +++ b/go/cmd/mesh-compose/main.go @@ -4,13 +4,15 @@ // it answers what that commit's composition answers, so a program that links one mesh-sdk can have another's judged. // // mesh-compose < request.json > verdict.json +// mesh-compose -request request.json > verdict.json // mesh-compose -catalogue [-node ]... [-depth resolve|compose] [-declarations] [< request.json] // // -catalogue reads /modules/*/module.json and a module.json at 's root as the first case's manifests (a // checkout or `git archive` of the catalogue repository at a ref: the command fetches nothing). -node adds a machine to // compose; -declarations answers each machine's declared paths with their owners and modes. // -// It opens no network connection, bus, store or file beyond its standard input and the -catalogue directory, and reads +// It opens no network connection, bus, store or file beyond its standard input, the -request file and the -catalogue +// directory, and reads // no environment. Exit codes: 0 the verdict is written (a machine that does not compose is a verdict, not an error); // 1 the request cannot be read or run, said on standard error; 2 the flags are wrong. package main @@ -47,6 +49,7 @@ func run(args []string, stdin io.Reader, stdout, stderr io.Writer) int { fs.Var(&machines, "node", "a machine to compose (repeatable)") depth := fs.String("depth", "", "resolve or compose (default: the request's, else compose)") declarations := fs.Bool("declarations", false, "answer each composed machine's declared paths, owners and modes") + requestFile := fs.String("request", "", "read the request from this file instead of standard input (a run step that hands no standard input)") if err := fs.Parse(args); err != nil { return 2 } @@ -54,9 +57,15 @@ func run(args []string, stdin io.Reader, stdout, stderr io.Writer) int { fmt.Fprintf(stderr, "mesh-compose: unexpected arguments %q\n", fs.Args()) return 2 } - raw, err := io.ReadAll(stdin) + var raw []byte + var err error + if *requestFile != "" { + raw, err = os.ReadFile(*requestFile) + } else { + raw, err = io.ReadAll(stdin) + } if err != nil { - fmt.Fprintln(stderr, "mesh-compose: standard input cannot be read:", err) + fmt.Fprintln(stderr, "mesh-compose: the request cannot be read:", err) return 1 } var req composer.Request diff --git a/go/cmd/mesh-compose/main_test.go b/go/cmd/mesh-compose/main_test.go index 358e743..a878b74 100644 --- a/go/cmd/mesh-compose/main_test.go +++ b/go/cmd/mesh-compose/main_test.go @@ -184,6 +184,24 @@ func TestACatalogueCheckoutIsComposedForANode(t *testing.T) { } } +// A run step hands the command no standard input: the request is read from the file -request names, and a file that is +// not there is a request it cannot read. +func TestTheRequestIsReadFromAFile(t *testing.T) { + file := filepath.Join(t.TempDir(), "request.json") + r := composer.Request{V: composer.Version, Cases: []composer.Case{{Name: "only", Manifests: map[string]composer.Manifest{ + "demo": {Raw: json.RawMessage(`{"module":"demo","version":"1"}`)}}}}} + if err := os.WriteFile(file, encode(t, r), 0o600); err != nil { + t.Fatal(err) + } + code, v, stderr := runJSON(t, []string{"-request", file}, nil) + if code != 0 || len(v.Cases) != 1 || !v.Cases[0].Manifests["demo"].Accepted { + t.Fatalf("exit %d (%s), verdict %+v", code, stderr, v) + } + if code, _, stderr := runJSON(t, []string{"-request", file + ".gone"}, nil); code != 1 || !strings.Contains(stderr, "cannot be read") { + t.Errorf("a request file that is not there: exit %d, %q", code, stderr) + } +} + // The exit codes: 1 for a request that cannot be read or run, 2 for wrong flags, each said on standard error. func TestTheExitCodesSayWhatFailed(t *testing.T) { unfilled := encode(t, composer.Request{V: composer.Version, Cases: []composer.Case{{Name: "c", Manifests: map[string]composer.Manifest{"x": {Digest: "abc"}}}}}) From dd0d156b9edc7130e859d3b23d8a886950ed94e7 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 24 Oct 2026 04:25:12 +0200 Subject: [PATCH 6/9] Wait for the third run to start, not only for its take, so the capacity test does not race under a loaded suite --- go/buildstep/pull_test.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/go/buildstep/pull_test.go b/go/buildstep/pull_test.go index 6d0b326..ddfd33a 100644 --- a/go/buildstep/pull_test.go +++ b/go/buildstep/pull_test.go @@ -127,11 +127,8 @@ func TestPullHoldsAtMostItsCapacity(t *testing.T) { } release <- struct{}{} waitFor(t, "the third taken", func() bool { return src.Waiting(delivery.AskDelivery) == 0 }) - mu.Lock() - if started != 3 { - t.Errorf("started %d, want 3", started) - } - mu.Unlock() + // Taken from the source comes a moment before its run starts: wait for the start, not only the take. + waitFor(t, "the third started", func() bool { mu.Lock(); defer mu.Unlock(); return started == 3 }) close(release) waitFor(t, "all acked", func() bool { a, _, _ := src.Fates(); return len(a) == 3 }) } From d9fa5536a82e092215910d7d26edb80df6abdb28 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 24 Oct 2026 04:38:58 +0200 Subject: [PATCH 7/9] Mark the packages whose tests read files outside their directory, found by the review (issue 1935) --- go/cmd/mesh-compose/reads-outside | 1 + go/internal/beside/reads-outside | 1 + go/packageguard/reads-outside | 1 + go/reads-outside | 1 + 4 files changed, 4 insertions(+) create mode 100644 go/cmd/mesh-compose/reads-outside create mode 100644 go/internal/beside/reads-outside create mode 100644 go/packageguard/reads-outside create mode 100644 go/reads-outside diff --git a/go/cmd/mesh-compose/reads-outside b/go/cmd/mesh-compose/reads-outside new file mode 100644 index 0000000..f6b2d87 --- /dev/null +++ b/go/cmd/mesh-compose/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/go/internal/beside/reads-outside b/go/internal/beside/reads-outside new file mode 100644 index 0000000..f6b2d87 --- /dev/null +++ b/go/internal/beside/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/go/packageguard/reads-outside b/go/packageguard/reads-outside new file mode 100644 index 0000000..f6b2d87 --- /dev/null +++ b/go/packageguard/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/go/reads-outside b/go/reads-outside new file mode 100644 index 0000000..f6b2d87 --- /dev/null +++ b/go/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). From 19c1d35939bc1abe80117a6cabaec571d55771b0 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 24 Oct 2026 14:11:20 +0200 Subject: [PATCH 8/9] Name the pinned mesh-sdk version when an SDK command is built from a reader's tree, not a bare (devel): the compose job builds mesh-compose there and its verdict must name the reader's SDK (ADR 0482 clause 5) --- go/sdkversion/sdkversion.go | 9 ++++++++- go/sdkversion/sdkversion_test.go | 12 ++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/go/sdkversion/sdkversion.go b/go/sdkversion/sdkversion.go index 46a705a..3cbb82e 100644 --- a/go/sdkversion/sdkversion.go +++ b/go/sdkversion/sdkversion.go @@ -24,7 +24,11 @@ func Version() string { // FromBuildInfo reads the SDK's version out of a build info. A dependency replaced by another module version is that // version; replaced by a local path it is the path, never empty. When the SDK is the main module it is "(devel)", -// followed by a space and the vcs.revision when the build stamped one. Without the SDK in the build it is "unknown". +// followed by a space and the vcs.revision when the build stamped one. An SDK command built from another module that +// requires the SDK (`go build git.novox.be/novox/mesh-sdk/go/cmd/mesh-compose` in a reader's tree, as mesh-build-go's +// compose job builds it: novox/hq ADR 0482 clause 5) has the SDK as its main module at the version that module pins, +// read from the module cache with no vcs stamp: it is that version, never a bare "(devel)" that names no SDK. Without +// the SDK in the build it is "unknown". func FromBuildInfo(bi *debug.BuildInfo) string { if bi == nil { return "unknown" @@ -35,6 +39,9 @@ func FromBuildInfo(bi *debug.BuildInfo) string { return "(devel) " + s.Value } } + if v := bi.Main.Version; v != "" && v != "(devel)" { + return v + } return "(devel)" } for _, d := range bi.Deps { diff --git a/go/sdkversion/sdkversion_test.go b/go/sdkversion/sdkversion_test.go index 7a4b04d..3cc60a2 100644 --- a/go/sdkversion/sdkversion_test.go +++ b/go/sdkversion/sdkversion_test.go @@ -42,6 +42,18 @@ func TestTheMainModuleIsDevelWithItsRevision(t *testing.T) { } } +// The shape go gives `go build ` run in a module that requires the SDK (captured with go1.27 from +// mesh-compose built in a reader's tree at bc0c25e51052: "mod git.novox.be/novox/mesh-sdk/go v0.1.14-…-bc0c25e51052", +// no vcs settings): the SDK is the main module at the pinned version, which names the SDK the binary carries. +func TestAnSDKCommandBuiltFromAReaderIsThePinnedVersion(t *testing.T) { + bi := &debug.BuildInfo{Path: sdkPath + "/cmd/mesh-compose", + Main: debug.Module{Path: sdkPath, Version: "v0.1.14-0.20261024112544-bc0c25e51052"}, + Settings: []debug.BuildSetting{{Key: "-trimpath", Value: "true"}, {Key: "CGO_ENABLED", Value: "0"}}} + if got := FromBuildInfo(bi); got != "v0.1.14-0.20261024112544-bc0c25e51052" { + t.Fatalf("got %q", got) + } +} + func TestNoBuildInfoIsUnknown(t *testing.T) { if got := FromBuildInfo(nil); got != "unknown" { t.Fatalf("got %q", got) From bfc3883af980d767ea3d889aa72e7a236686a25b Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 24 Oct 2026 14:57:10 +0200 Subject: [PATCH 9/9] =?UTF-8?q?Read=20a=20manifest's=20fallback=20to=20a?= =?UTF-8?q?=20predefined=20key=20in=20the=20composer=20as=20the=20indexer?= =?UTF-8?q?=20reads=20it,=20or=20against=20the=20list=20a=20request=20or?= =?UTF-8?q?=20-catalogue=20gives:=20a=20run=20never=20holds=20the=20catalo?= =?UTF-8?q?gue's=20predefined.json,=20and=20the=20composition=20refused=20?= =?UTF-8?q?pacman=20and=20xterm=20for=20it=20(ADR=200348=20=C2=A77)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go/cmd/mesh-compose/main.go | 10 +++++++++ go/cmd/mesh-compose/main_test.go | 38 +++++++++++++++++++++++++++++++- go/composer/composer.go | 23 +++++++++++++++---- go/composer/composer_test.go | 31 ++++++++++++++++++++++++++ 4 files changed, 97 insertions(+), 5 deletions(-) diff --git a/go/cmd/mesh-compose/main.go b/go/cmd/mesh-compose/main.go index cc0211d..3168bde 100644 --- a/go/cmd/mesh-compose/main.go +++ b/go/cmd/mesh-compose/main.go @@ -28,6 +28,7 @@ import ( "sort" "strings" + "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/composer" ) @@ -96,6 +97,15 @@ func run(args []string, stdin io.Reader, stdout, stderr io.Writer) int { req.Cases[0].Manifests[name] = m } } + // The catalogue's list of predefined keys is the one its manifests' fallbacks are judged against (ADR 0348 §7). + if req.Predefined == nil { + list, err := catalogue.ReadPredefined(*catalogueDir) + if err != nil { + fmt.Fprintln(stderr, "mesh-compose:", err) + return 1 + } + req.Predefined = list + } } req.Machines = append(req.Machines, machines...) if *depth != "" { diff --git a/go/cmd/mesh-compose/main_test.go b/go/cmd/mesh-compose/main_test.go index a878b74..c68ff93 100644 --- a/go/cmd/mesh-compose/main_test.go +++ b/go/cmd/mesh-compose/main_test.go @@ -114,7 +114,8 @@ func TestTheCommandAnswersWhatTheLinkedCompositionAnswers(t *testing.T) { in := meshOf() in.Catalogue = map[string]catalogue.Manifest{} for name, m := range r.Cases[0].Manifests { - parsed, err := catalogue.ParseManifest(m.Raw) + // A request without a list of predefined keys reads a fallback as the indexer does (Request.Predefined). + parsed, err := catalogue.ParseManifestForInputs(m.Raw) got := v.Cases[0].Manifests[name] if (err == nil) != got.Accepted || (err != nil && err.Error() != got.Refused) { t.Errorf("%s: the manifest of %s: linked %v, command %+v", depth, name, err, got) @@ -309,3 +310,38 @@ func TestTheCommandRunsWithNoNetwork(t *testing.T) { sort.Strings(names) t.Logf("composed with no network: %v", names) } + +// -catalogue judges a fallback to a predefined key against the predefined.json at the checkout's root (ADR 0348 §7): +// xterm (mesh-catalog at 835bf22b) falls back to "font-size", accepted where the root lists it and refused where the +// root's list does not. +func TestACatalogueCheckoutJudgesFallbacksAgainstItsPredefinedKeys(t *testing.T) { + raw, err := os.ReadFile("../../catalogue/testdata/catalogue-835bf22b-xterm.module.json") + if err != nil { + t.Fatal(err) + } + for _, c := range []struct { + list string + want bool + }{ + {`{"font-size": {"type": "number", "description": "the desktop's font size"}}`, true}, + {`{"font-family": {"type": "string", "description": "the desktop's font"}}`, false}, + } { + dir := t.TempDir() + if err := os.MkdirAll(filepath.Join(dir, "modules", "xterm"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "modules", "xterm", "module.json"), raw, 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, catalogue.PredefinedFile), []byte(c.list), 0o600); err != nil { + t.Fatal(err) + } + code, v, stderr := runJSON(t, []string{"-catalogue", dir}, nil) + if code != 0 { + t.Fatalf("exit %d: %s", code, stderr) + } + if got := v.Cases[0].Manifests["xterm"]; got.Accepted != c.want { + t.Errorf("predefined.json %s: xterm %+v, want accepted %v", c.list, got, c.want) + } + } +} diff --git a/go/composer/composer.go b/go/composer/composer.go index 5ab962a..a456a1a 100644 --- a/go/composer/composer.go +++ b/go/composer/composer.go @@ -55,6 +55,12 @@ type Request struct { HoldingsFromClaims bool `json:"holdings_from_claims,omitempty"` // Declarations asks each composed machine's declared paths (DepthCompose only). Declarations bool `json:"declarations,omitempty"` + // Predefined is the list of predefined keys at the catalogue's root (predefined.json, novox/hq ADR 0348 §7) that + // every config value falling back to a key is judged against, when the asker has it (mesh-compose -catalogue + // reads it). Without it a fallback is read as catalogue.ParseManifestForInputs reads it: against the list the mesh + // published when this process holds one, else each key as its value declares it, for the merge gate judges the + // fallback in the catalogue and a manifest that only carries one is not refused here. + Predefined catalogue.PredefinedList `json:"predefined,omitempty"` } // Case is one catalogue composed: every module's manifest by name, and per node the manifests kept at the build it runs. @@ -209,13 +215,22 @@ func Run(r Request) (Verdict, error) { return v, nil } -func parse(m map[string]Manifest, reads map[string]ManifestRead) (map[string]catalogue.Manifest, error) { +// readManifest reads one manifest as this run reads every one: against the request's list of predefined keys when it +// has one, else as the indexer and the builder read it (Request.Predefined). +func readManifest(raw []byte, predefined catalogue.PredefinedList) (catalogue.Manifest, error) { + if predefined != nil { + return catalogue.ParseManifestWith(raw, predefined) + } + return catalogue.ParseManifestForInputs(raw) +} + +func parse(m map[string]Manifest, reads map[string]ManifestRead, predefined catalogue.PredefinedList) (map[string]catalogue.Manifest, error) { out := map[string]catalogue.Manifest{} for name, man := range m { if len(man.Raw) == 0 { return nil, fmt.Errorf("%w: %s (digest %s)", ErrUnfilled, name, man.Digest) } - parsed, err := catalogue.ParseManifest(man.Raw) + parsed, err := readManifest(man.Raw, predefined) if reads != nil { if err != nil { reads[name] = ManifestRead{Refused: err.Error()} @@ -232,7 +247,7 @@ func parse(m map[string]Manifest, reads map[string]ManifestRead) (map[string]cat func runCase(r Request, c Case, depth string) (CaseVerdict, error) { cv := CaseVerdict{Name: c.Name, Manifests: map[string]ManifestRead{}} - shelf, err := parse(c.Manifests, cv.Manifests) + shelf, err := parse(c.Manifests, cv.Manifests, r.Predefined) if err != nil { return CaseVerdict{}, err } @@ -243,7 +258,7 @@ func runCase(r Request, c Case, depth string) (CaseVerdict, error) { for i := range in.Nodes { in.Nodes[i].Kept = nil if kept := c.Kept[in.Nodes[i].Name]; len(kept) > 0 { - k, err := parse(kept, nil) + k, err := parse(kept, nil, r.Predefined) if err != nil { return CaseVerdict{}, err } diff --git a/go/composer/composer_test.go b/go/composer/composer_test.go index 175894a..5e26900 100644 --- a/go/composer/composer_test.go +++ b/go/composer/composer_test.go @@ -3,6 +3,7 @@ package composer import ( "encoding/json" "errors" + "os" "reflect" "strings" "testing" @@ -104,3 +105,33 @@ func TestARequestItCannotRunIsRefused(t *testing.T) { t.Fatalf("a request with no machines: %+v, %v", v, err) } } + +// A manifest whose config value falls back to a predefined key (novox/hq ADR 0348 §7) is read, not refused for the +// list of predefined keys at the catalogue's root that a run is never handed with its manifests: the merge gate judges +// the fallback in the catalogue. xterm's module.json from mesh-catalog at 835bf22b falls back to "font-size"; a group +// touching xterm or pacman failed its composition on this (group check 18df81380db3). A list in the request is the one +// judged against, so a fallback it does not name is still refused. +func TestAManifestFallingBackToAPredefinedKeyIsRead(t *testing.T) { + raw, err := os.ReadFile("../catalogue/testdata/catalogue-835bf22b-xterm.module.json") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(raw), `"predefined"`) { + t.Fatal("the fixture names no predefined key") + } + r := Request{V: Version, Cases: []Case{{Name: "change", Manifests: map[string]Manifest{"xterm": {Raw: raw}}}}} + v, err := Run(r) + if err != nil { + t.Fatal(err) + } + if got := v.Cases[0].Manifests["xterm"]; !got.Accepted { + t.Fatalf("xterm: %+v, want accepted", got) + } + r.Predefined = catalogue.PredefinedList{"something-else": {Type: "string", Description: "x"}} + if v, err = Run(r); err != nil { + t.Fatal(err) + } + if got := v.Cases[0].Manifests["xterm"]; got.Accepted || !strings.Contains(got.Refused, "not on the list of predefined keys") { + t.Fatalf("xterm judged against a list without its key: %+v, want refused naming the list", got) + } +}