From 20d18eb4f5c38029f2d3e41560eb8d49e61568f6 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 24 Oct 2026 03:51:35 +0200 Subject: [PATCH] Give mesh-sdk a composer command, so a program linking one mesh-sdk can have another commit's composition judged (issue 1621) --- go/cmd/mesh-compose/main.go | 144 ++++++++++++++ go/cmd/mesh-compose/main_test.go | 293 +++++++++++++++++++++++++++ go/composer/composer.go | 332 +++++++++++++++++++++++++++++++ go/composer/composer_test.go | 106 ++++++++++ 4 files changed, 875 insertions(+) create mode 100644 go/cmd/mesh-compose/main.go create mode 100644 go/cmd/mesh-compose/main_test.go create mode 100644 go/composer/composer.go create mode 100644 go/composer/composer_test.go diff --git a/go/cmd/mesh-compose/main.go b/go/cmd/mesh-compose/main.go new file mode 100644 index 0000000..e871b0a --- /dev/null +++ b/go/cmd/mesh-compose/main.go @@ -0,0 +1,144 @@ +// mesh-compose is mesh-sdk's composer command (novox/hq ADR 0482 clause 4, issue 1621): it reads a composer.Request on +// standard input, validates every manifest with this mesh-sdk's reader, composes the machines named in each case and +// writes one composer.Verdict on standard output. Built at any commit of mesh-sdk (or at the mesh-sdk a module pins), +// it answers what that commit's composition answers, so a program that links one mesh-sdk can have another's judged. +// +// mesh-compose < request.json > verdict.json +// mesh-compose -catalogue [-node ]... [-depth resolve|compose] [-declarations] [< request.json] +// +// -catalogue reads /modules/*/module.json and a module.json at 's root as the first case's manifests (a +// checkout or `git archive` of the catalogue repository at a ref: the command fetches nothing). -node adds a machine to +// compose; -declarations answers each machine's declared paths with their owners and modes. +// +// It opens no network connection, bus, store or file beyond its standard input and the -catalogue directory, and reads +// no environment. Exit codes: 0 the verdict is written (a machine that does not compose is a verdict, not an error); +// 1 the request cannot be read or run, said on standard error; 2 the flags are wrong. +package main + +import ( + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + + "git.novox.be/novox/mesh-sdk/go/composer" +) + +func main() { + os.Exit(run(os.Args[1:], os.Stdin, os.Stdout, os.Stderr)) +} + +type nodes []string + +func (n *nodes) String() string { return strings.Join(*n, ",") } +func (n *nodes) Set(v string) error { *n = append(*n, v); return nil } + +// run is the command: its arguments, standard input, output and error; it answers the exit code. +func run(args []string, stdin io.Reader, stdout, stderr io.Writer) int { + fs := flag.NewFlagSet("mesh-compose", flag.ContinueOnError) + fs.SetOutput(stderr) + catalogueDir := fs.String("catalogue", "", "a catalogue checkout: /modules/*/module.json are the first case's manifests") + var machines nodes + fs.Var(&machines, "node", "a machine to compose (repeatable)") + depth := fs.String("depth", "", "resolve or compose (default: the request's, else compose)") + declarations := fs.Bool("declarations", false, "answer each composed machine's declared paths, owners and modes") + if err := fs.Parse(args); err != nil { + return 2 + } + if fs.NArg() > 0 { + fmt.Fprintf(stderr, "mesh-compose: unexpected arguments %q\n", fs.Args()) + return 2 + } + raw, err := io.ReadAll(stdin) + if err != nil { + fmt.Fprintln(stderr, "mesh-compose: standard input cannot be read:", err) + return 1 + } + var req composer.Request + if strings.TrimSpace(string(raw)) == "" { + if *catalogueDir == "" { + fmt.Fprintln(stderr, "mesh-compose: no request on standard input and no -catalogue") + return 1 + } + req = composer.Request{V: composer.Version} + } else if req, err = composer.Decode(raw); err != nil { + fmt.Fprintln(stderr, "mesh-compose:", err) + return 1 + } + if *catalogueDir != "" { + read, err := readCatalogue(*catalogueDir) + if err != nil { + fmt.Fprintln(stderr, "mesh-compose:", err) + return 1 + } + if len(req.Cases) == 0 { + req.Cases = []composer.Case{{Name: "catalogue", Manifests: map[string]composer.Manifest{}}} + } + if req.Cases[0].Manifests == nil { + req.Cases[0].Manifests = map[string]composer.Manifest{} + } + for name, m := range read { + if _, given := req.Cases[0].Manifests[name]; !given { + req.Cases[0].Manifests[name] = m + } + } + } + req.Machines = append(req.Machines, machines...) + if *depth != "" { + req.Depth = *depth + } + if *declarations { + req.Declarations = true + } + v, err := composer.Run(req) + if err != nil { + fmt.Fprintln(stderr, "mesh-compose:", err) + return 1 + } + enc := json.NewEncoder(stdout) + if err := enc.Encode(v); err != nil { + fmt.Fprintln(stderr, "mesh-compose: the verdict cannot be written:", err) + return 1 + } + return 0 +} + +// readCatalogue reads every module.json of a catalogue checkout by its module's name: the root's and each of +// modules/*/. A manifest whose name cannot be read is kept under its directory's name, so the reader's refusal is said. +func readCatalogue(dir string) (map[string]composer.Manifest, error) { + paths, err := filepath.Glob(filepath.Join(dir, "modules", "*", "module.json")) + if err != nil { + return nil, err + } + if _, err := os.Stat(filepath.Join(dir, "module.json")); err == nil { + paths = append(paths, filepath.Join(dir, "module.json")) + } + if len(paths) == 0 { + return nil, fmt.Errorf("the catalogue %s holds no module.json", dir) + } + sort.Strings(paths) + out := map[string]composer.Manifest{} + for _, p := range paths { + raw, err := os.ReadFile(p) + if err != nil { + return nil, err + } + var probe struct { + Module string `json:"module"` + } + name := filepath.Base(filepath.Dir(p)) + if json.Unmarshal(raw, &probe) == nil && probe.Module != "" { + name = probe.Module + } + if _, twice := out[name]; twice { + return nil, errors.New("the catalogue " + dir + " defines " + name + " twice") + } + out[name] = composer.Manifest{Raw: raw} + } + return out, nil +} diff --git a/go/cmd/mesh-compose/main_test.go b/go/cmd/mesh-compose/main_test.go new file mode 100644 index 0000000..358e743 --- /dev/null +++ b/go/cmd/mesh-compose/main_test.go @@ -0,0 +1,293 @@ +package main + +import ( + "bytes" + "encoding/json" + "go/build" + "os" + "os/exec" + "path/filepath" + "reflect" + "runtime" + "sort" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/compose" + "git.novox.be/novox/mesh-sdk/go/composer" + "git.novox.be/novox/mesh-sdk/go/internal/beside" +) + +// captured is every manifest of the catalogue captured in testdata/beside (internal/beside), raw, by module name. +func captured(t *testing.T) map[string]composer.Manifest { + t.Helper() + read, err := readCatalogue(filepath.Dir(beside.Catalogue(t))) + if err != nil { + t.Fatal(err) + } + if len(read) < 100 { + t.Fatalf("the captured catalogue holds %d manifests, fewer than the catalogue's", len(read)) + } + return read +} + +// meshOf is a mesh of three machines over the captured catalogue, each assigned real modules, the input the assigner +// composes with (nodes and their assignments, holdings from the claims): anchor runs the bus and the store, laptop a +// desktop's modules, ace a module whose requirement is provided elsewhere. +func meshOf() compose.Input { + return compose.Input{Nodes: []compose.Node{ + {Name: "anchor", Assigned: []string{"nats", "postgres"}}, + {Name: "laptop", Assigned: []string{"sudo", "zsh"}}, + {Name: "ace", Assigned: []string{"mesh-reads", "nats"}}, + }} +} + +func request(t *testing.T, depth string) composer.Request { + t.Helper() + return composer.Request{V: composer.Version, Input: meshOf(), Depth: depth, HoldingsFromClaims: true, + Machines: []string{"anchor", "laptop", "ace"}, + Cases: []composer.Case{{Name: "trunk", Manifests: captured(t)}}} +} + +func runJSON(t *testing.T, args []string, in []byte) (int, composer.Verdict, string) { + t.Helper() + var out, errOut bytes.Buffer + code := run(args, bytes.NewReader(in), &out, &errOut) + var v composer.Verdict + if code == 0 { + if err := json.Unmarshal(out.Bytes(), &v); err != nil { + t.Fatalf("the verdict is not JSON: %v\n%s", err, out.String()) + } + } + return code, v, errOut.String() +} + +func encode(t *testing.T, r composer.Request) []byte { + t.Helper() + raw, err := json.Marshal(r) + if err != nil { + t.Fatal(err) + } + return raw +} + +// 1621/AC1: the command reads the plain input and the manifests on standard input, validates every manifest, composes +// the machines named and writes one verdict: per machine composes or the reason, per manifest accepted or the field +// refused. +func TestTheCommandWritesOneVerdictPerManifestAndMachine(t *testing.T) { + r := request(t, composer.DepthResolve) + r.Cases[0].Manifests["broken"] = composer.Manifest{Raw: json.RawMessage(`{"module":"broken","version":"1","shares":["x"]}`)} + code, v, stderr := runJSON(t, nil, encode(t, r)) + if code != 0 { + t.Fatalf("exit %d: %s", code, stderr) + } + if len(v.Cases) != 1 || v.Cases[0].Name != "trunk" || v.SDK == "" { + t.Fatalf("verdict = %+v", v) + } + c := v.Cases[0] + if got := c.Manifests["broken"]; got.Accepted || !strings.Contains(got.Refused, `"shares"`) { + t.Errorf("the manifest with an unknown field reads %+v, want refused naming the field", got) + } + if got := c.Manifests["nats"]; !got.Accepted { + t.Errorf("nats reads %+v, want accepted", got) + } + if len(c.Machines) != 3 { + t.Fatalf("machines = %v, want the three named", c.Machines) + } + for name, m := range c.Machines { + if !m.Composes && m.Reason == "" { + t.Errorf("%s does not compose and says no reason", name) + } + } +} + +// 1621/AC3: over the captured catalogue and a mesh of three machines, the command's verdict equals, machine for +// machine and manifest for manifest, what the linked reader, resolution and composition answer, at both depths. +func TestTheCommandAnswersWhatTheLinkedCompositionAnswers(t *testing.T) { + for _, depth := range []string{composer.DepthResolve, composer.DepthCompose} { + r := request(t, depth) + code, v, stderr := runJSON(t, nil, encode(t, r)) + if code != 0 { + t.Fatalf("%s: exit %d: %s", depth, code, stderr) + } + in := meshOf() + in.Catalogue = map[string]catalogue.Manifest{} + for name, m := range r.Cases[0].Manifests { + parsed, err := catalogue.ParseManifest(m.Raw) + got := v.Cases[0].Manifests[name] + if (err == nil) != got.Accepted || (err != nil && err.Error() != got.Refused) { + t.Errorf("%s: the manifest of %s: linked %v, command %+v", depth, name, err, got) + } + if err == nil { + in.Catalogue[name] = parsed + } + } + in.Holdings = composer.HoldingsOf(in.Catalogue, in.Nodes) + composed := 0 + for _, node := range r.Machines { + got := v.Cases[0].Machines[node] + var want composer.Machine + if depth == composer.DepthResolve { + if _, err := compose.Resolve(in, node); err != nil { + want = composer.Machine{Reason: err.Error()} + } else { + want = composer.Machine{Composes: true} + } + } else { + d, problems, err := compose.Compose(in, node) + switch { + case err != nil: + want = composer.Machine{Reason: err.Error()} + default: + want = composer.Machine{Composes: len(problems) == 0, Problems: problems, Refusals: d.Refusals()} + if len(problems) > 0 { + want.Reason = "the node-engine's validator refuses the declaration" + } + } + } + if !reflect.DeepEqual(got, want) { + t.Errorf("%s: %s: command %+v, linked %+v", depth, node, got, want) + } + if got.Composes { + composed++ + } else { + t.Logf("%s: %s does not compose: %s %v", depth, node, got.Reason, got.Problems) + } + } + t.Logf("%s: %d of %d machines compose", depth, composed, len(r.Machines)) + } +} + +// 1621 note 6: a person composes a catalogue checkout (repository@ref checked out or archived: the command fetches +// nothing) for a node and reads its declaration: each path with its kind, owner and mode. +func TestACatalogueCheckoutIsComposedForANode(t *testing.T) { + dir := t.TempDir() + if err := os.MkdirAll(filepath.Join(dir, "modules", "demo"), 0o755); err != nil { + t.Fatal(err) + } + manifest := `{"module": "demo", "version": "1", "resources": [{"id": "state", "type": "directory", ` + + `"path": "/var/lib/demo", "mode": "0750", "owner": "root"}]}` + if err := os.WriteFile(filepath.Join(dir, "modules", "demo", "module.json"), []byte(manifest), 0o644); err != nil { + t.Fatal(err) + } + in := encode(t, composer.Request{V: composer.Version, Input: compose.Input{Nodes: []compose.Node{{Name: "laptop", Assigned: []string{"demo"}}}}}) + code, v, stderr := runJSON(t, []string{"-catalogue", dir, "-node", "laptop", "-declarations"}, in) + if code != 0 { + t.Fatalf("exit %d: %s", code, stderr) + } + m := v.Cases[0].Machines["laptop"] + want := []composer.Path{{Path: "/var/lib/demo", Kind: "directory", Owner: "root", Mode: "0750", Module: ""}} + if !m.Composes || len(m.Declared) != 1 || m.Declared[0].Path != want[0].Path || m.Declared[0].Mode != "0750" || + m.Declared[0].Owner != "root" || m.Declared[0].Kind != "directory" { + t.Fatalf("laptop = %+v, want it composed declaring %+v", m, want) + } +} + +// The exit codes: 1 for a request that cannot be read or run, 2 for wrong flags, each said on standard error. +func TestTheExitCodesSayWhatFailed(t *testing.T) { + unfilled := encode(t, composer.Request{V: composer.Version, Cases: []composer.Case{{Name: "c", Manifests: map[string]composer.Manifest{"x": {Digest: "abc"}}}}}) + for _, c := range []struct { + name string + args []string + in string + code int + says string + }{ + {"not JSON", nil, "{", 1, "not JSON"}, + {"another version", nil, `{"v":2}`, 1, "version 2"}, + {"a digest left unfilled", nil, string(unfilled), 1, "digest"}, + {"an unknown depth", nil, `{"v":1,"depth":"deep"}`, 1, "depth"}, + {"nothing to read", nil, "", 1, "no request"}, + {"a wrong flag", []string{"-nope"}, "", 2, "nope"}, + {"an argument", []string{"extra"}, "", 2, "unexpected"}, + } { + var out, errOut bytes.Buffer + if got := run(c.args, strings.NewReader(c.in), &out, &errOut); got != c.code || !strings.Contains(errOut.String(), c.says) { + t.Errorf("%s: exit %d saying %q, want %d saying %q", c.name, got, errOut.String(), c.code, c.says) + } + } +} + +// sdkPackages is every package of this module the command imports, directly or not, with the packages each imports. +func sdkPackages(t *testing.T) map[string][]string { + t.Helper() + _, file, _, _ := runtime.Caller(0) + root := filepath.Join(filepath.Dir(file), "..", "..") + const module = "git.novox.be/novox/mesh-sdk/go" + out := map[string][]string{} + var walk func(path string) + walk = func(path string) { + if _, done := out[path]; done { + return + } + rel := strings.TrimPrefix(strings.TrimPrefix(path, module), "/") + pkg, err := build.Default.ImportDir(filepath.Join(root, rel), 0) + if err != nil { + t.Fatalf("%s: %v", path, err) + } + out[path] = pkg.Imports + for _, imp := range pkg.Imports { + if strings.HasPrefix(imp, module+"/") { + walk(imp) + } + } + } + walk(composer.Package) + return out +} + +// 1621/AC2 and AC4: the command imports, through every package of mesh-sdk it reaches, only the standard library and +// mesh-sdk's own packages (so `go build` from mesh-sdk's module needs nothing beyond its lock), and none that reaches +// a bus, a store, a network service or another process. +func TestTheCommandImportsNoBusStoreNetworkOrProcess(t *testing.T) { + forbidden := []string{"net/http", "net/rpc", "net/smtp", "os/exec", "database/sql", "plugin", "syscall"} + for path, imports := range sdkPackages(t) { + for _, imp := range imports { + std := !strings.Contains(strings.SplitN(imp, "/", 2)[0], ".") + if !std && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-sdk/go/") { + t.Errorf("%s imports %s, which is neither the standard library nor mesh-sdk", path, imp) + } + for _, f := range forbidden { + if imp == f { + t.Errorf("%s imports %s", path, imp) + } + } + } + } +} + +// 1621/AC2: run with an empty network namespace, the command still writes its verdict: it needs no network. The test +// runs its own binary again under `unshare --net` as an unprivileged user's namespace; where the kernel or the sandbox +// allows no new namespace (a build module's run step is already one with no network), it says so and skips. +func TestTheCommandRunsWithNoNetwork(t *testing.T) { + if os.Getenv("MESH_COMPOSE_HELPER") == "1" { + os.Exit(run(nil, os.Stdin, os.Stdout, os.Stderr)) + } + unshare, err := exec.LookPath("unshare") + if err != nil { + t.Skip("no unshare on this machine, so no empty network namespace can be made here") + } + if out, err := exec.Command(unshare, "--net", "--map-root-user", "true").CombinedOutput(); err != nil { + t.Skipf("this machine makes no unprivileged network namespace (%v: %s); a run step's own has no network", err, out) + } + r := request(t, composer.DepthResolve) + cmd := exec.Command(unshare, "--net", "--map-root-user", os.Args[0], "-test.run=^TestTheCommandRunsWithNoNetwork$") + cmd.Env = []string{"MESH_COMPOSE_HELPER=1"} + cmd.Stdin = bytes.NewReader(encode(t, r)) + var out, errOut bytes.Buffer + cmd.Stdout, cmd.Stderr = &out, &errOut + if err := cmd.Run(); err != nil { + t.Fatalf("the command in an empty network namespace: %v: %s", err, errOut.String()) + } + var v composer.Verdict + if err := json.Unmarshal(out.Bytes(), &v); err != nil || len(v.Cases) != 1 || len(v.Cases[0].Machines) != 3 { + t.Fatalf("no verdict from the command with no network: %v\n%s", err, out.String()) + } + names := make([]string, 0, len(v.Cases[0].Machines)) + for n := range v.Cases[0].Machines { + names = append(names, n) + } + sort.Strings(names) + t.Logf("composed with no network: %v", names) +} diff --git a/go/composer/composer.go b/go/composer/composer.go new file mode 100644 index 0000000..5ab962a --- /dev/null +++ b/go/composer/composer.go @@ -0,0 +1,332 @@ +// Package composer is mesh-sdk's composition and validator as one function over a plain request (novox/hq ADR 0482 +// clause 4, issue 1621): what the command mesh-compose runs, so a program that links any one mesh-sdk can have the +// composition of another commit judged by building that commit's command and reading its verdict. +// +// **It reads its request and nothing else.** No bus, store, network, file or environment: the manifests come inside the +// request (raw, as their repository holds them), the rest of the mesh is a compose.Input, and the answer is a value. +// A manifest named only by its digest in the delivery indexer's bucket is filled before the run by Fill, by whoever +// holds the bus (the Go build module's compose job); Run refuses a digest left unfilled. +package composer + +import ( + "encoding/json" + "errors" + "fmt" + "slices" + "sort" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/compose" + "git.novox.be/novox/mesh-sdk/go/sdkversion" +) + +// Version is the version of a Request and of a Verdict; a reader refuses another. +const Version = 1 + +// Package is the import path of the command, which a compose job builds at a commit (in mesh-sdk's own module, or in +// a module that requires mesh-sdk, at the version it pins). +const Package = "git.novox.be/novox/mesh-sdk/go/cmd/mesh-compose" + +// The depths of a composition. +const ( + // DepthResolve resolves each machine's set over the rest of the mesh (compose.Resolve): what the assigner judges + // while the facts snapshot is not on the bus (issue 1461). + DepthResolve = "resolve" + // DepthCompose composes each machine's whole declaration and runs the node-engine's validator over it + // (compose.Compose). + DepthCompose = "compose" +) + +// Request is everything one run composes: the rest of the mesh, one or more cases of manifests composed over it, and +// the machines composed in each case. +type Request struct { + V int `json:"v"` + // Input is the composition's input without its catalogue (and without each node's kept manifests): the cases give + // those. A Catalogue or Kept it holds is ignored. + Input compose.Input `json:"input"` + // Cases are composed one after the other over Input, each with its own manifests; their names are unique. + Cases []Case `json:"cases"` + // Machines are the nodes composed in every case; none means the manifests are only read. + Machines []string `json:"machines,omitempty"` + // Depth is DepthResolve or DepthCompose; empty is DepthCompose. + Depth string `json:"depth,omitempty"` + // HoldingsFromClaims adds to Input's holdings every seat an assigned module claims, held on its node (the + // assigner's input, which has no holdings of record). + HoldingsFromClaims bool `json:"holdings_from_claims,omitempty"` + // Declarations asks each composed machine's declared paths (DepthCompose only). + Declarations bool `json:"declarations,omitempty"` +} + +// Case is one catalogue composed: every module's manifest by name, and per node the manifests kept at the build it runs. +type Case struct { + Name string `json:"name"` + Manifests map[string]Manifest `json:"manifests"` + Kept map[string]map[string]Manifest `json:"kept,omitempty"` +} + +// Manifest is a module's manifest as its repository holds it (Raw, the bytes), or the digest the delivery indexer keeps +// it under (Digest), which Fill replaces by the bytes. On the wire it is the manifest itself, or {"digest": "..."}. +type Manifest struct { + Raw json.RawMessage + Digest string +} + +// MarshalJSON writes the raw manifest, or {"digest": ...} when only the digest is known. +func (m Manifest) MarshalJSON() ([]byte, error) { + if len(m.Raw) > 0 { + return m.Raw, nil + } + return json.Marshal(map[string]string{"digest": m.Digest}) +} + +// UnmarshalJSON reads {"digest": "..."} (an object with that one key) as a digest, and any other value as the raw +// manifest. +func (m *Manifest) UnmarshalJSON(raw []byte) error { + var probe map[string]json.RawMessage + if json.Unmarshal(raw, &probe) == nil && len(probe) == 1 { + if d, ok := probe["digest"]; ok { + var digest string + if err := json.Unmarshal(d, &digest); err != nil { + return fmt.Errorf("a manifest's digest is not a string: %w", err) + } + *m = Manifest{Digest: digest} + return nil + } + } + *m = Manifest{Raw: slices.Clone(raw)} + return nil +} + +// Verdict is what one run answers: the mesh-sdk the command was built with, and each case's verdict in the order asked. +type Verdict struct { + V int `json:"v"` + SDK string `json:"sdk"` + Cases []CaseVerdict `json:"cases"` +} + +// CaseVerdict is one case's: every manifest read or refused, every machine composed or not. +type CaseVerdict struct { + Name string `json:"name"` + Manifests map[string]ManifestRead `json:"manifests"` + Machines map[string]Machine `json:"machines,omitempty"` +} + +// ManifestRead is one manifest as this mesh-sdk reads it: accepted, or refused with the reader's words. +type ManifestRead struct { + Accepted bool `json:"accepted"` + Refused string `json:"refused,omitempty"` +} + +// Machine is one machine composed: whether it composes (resolves, at DepthResolve), why not, the node-engine +// validator's problems and the composition's refusals (DepthCompose), and its declared paths when asked. +type Machine struct { + Composes bool `json:"composes"` + Reason string `json:"reason,omitempty"` + Problems []string `json:"problems,omitempty"` + Refusals []string `json:"refusals,omitempty"` + Declared []Path `json:"declared,omitempty"` +} + +// Path is one path a declaration puts on the machine, with its kind (the resource's type), owner, group and mode as +// declared, and the module it is of. +type Path struct { + Path string `json:"path"` + Kind string `json:"kind"` + Owner string `json:"owner,omitempty"` + Group string `json:"group,omitempty"` + Mode string `json:"mode,omitempty"` + Module string `json:"module,omitempty"` +} + +// ErrUnfilled is a manifest named by a digest that nobody replaced by its bytes. +var ErrUnfilled = errors.New("composer: a manifest is named by its digest and was not filled") + +// Decode reads a request; it refuses one that is not JSON or of another version. +func Decode(data []byte) (Request, error) { + var r Request + if err := json.Unmarshal(data, &r); err != nil { + return Request{}, fmt.Errorf("composer: the request is not JSON of its shape: %w", err) + } + if r.V != Version { + return Request{}, fmt.Errorf("composer: a request of version %d, this composer reads %d", r.V, Version) + } + return r, nil +} + +// Fill replaces every manifest named by its digest by the bytes get answers for that digest; a digest get cannot answer +// stops it, named. +func Fill(r *Request, get func(digest string) ([]byte, error)) error { + fill := func(m map[string]Manifest) error { + for name, man := range m { + if len(man.Raw) > 0 { + continue + } + raw, err := get(man.Digest) + if err != nil { + return fmt.Errorf("composer: the manifest of %s (digest %s): %w", name, man.Digest, err) + } + m[name] = Manifest{Raw: raw} + } + return nil + } + for _, c := range r.Cases { + if err := fill(c.Manifests); err != nil { + return err + } + for _, kept := range c.Kept { + if err := fill(kept); err != nil { + return err + } + } + } + return nil +} + +// Run composes every case of r and answers the verdict, its SDK this binary's mesh-sdk. An error is a request it +// cannot run (an unknown depth, a case named twice, a manifest left as a digest); a machine that does not compose is a +// verdict. +func Run(r Request) (Verdict, error) { + depth := r.Depth + if depth == "" { + depth = DepthCompose + } + if depth != DepthResolve && depth != DepthCompose { + return Verdict{}, fmt.Errorf("composer: the depth %q is not resolve or compose", r.Depth) + } + v := Verdict{V: Version, SDK: sdkversion.Version()} + seen := map[string]bool{} + for _, c := range r.Cases { + if seen[c.Name] { + return Verdict{}, fmt.Errorf("composer: the case %q is named twice", c.Name) + } + seen[c.Name] = true + cv, err := runCase(r, c, depth) + if err != nil { + return Verdict{}, err + } + v.Cases = append(v.Cases, cv) + } + return v, nil +} + +func parse(m map[string]Manifest, reads map[string]ManifestRead) (map[string]catalogue.Manifest, error) { + out := map[string]catalogue.Manifest{} + for name, man := range m { + if len(man.Raw) == 0 { + return nil, fmt.Errorf("%w: %s (digest %s)", ErrUnfilled, name, man.Digest) + } + parsed, err := catalogue.ParseManifest(man.Raw) + if reads != nil { + if err != nil { + reads[name] = ManifestRead{Refused: err.Error()} + } else { + reads[name] = ManifestRead{Accepted: true} + } + } + if err == nil { + out[name] = parsed + } + } + return out, nil +} + +func runCase(r Request, c Case, depth string) (CaseVerdict, error) { + cv := CaseVerdict{Name: c.Name, Manifests: map[string]ManifestRead{}} + shelf, err := parse(c.Manifests, cv.Manifests) + if err != nil { + return CaseVerdict{}, err + } + in := r.Input + in.Catalogue = shelf + in.Nodes = slices.Clone(in.Nodes) + in.Holdings = slices.Clone(in.Holdings) + for i := range in.Nodes { + in.Nodes[i].Kept = nil + if kept := c.Kept[in.Nodes[i].Name]; len(kept) > 0 { + k, err := parse(kept, nil) + if err != nil { + return CaseVerdict{}, err + } + in.Nodes[i].Kept = k + } + } + if r.HoldingsFromClaims { + in.Holdings = append(in.Holdings, HoldingsOf(shelf, in.Nodes)...) + } + if len(r.Machines) == 0 { + return cv, nil + } + cv.Machines = map[string]Machine{} + for _, node := range r.Machines { + cv.Machines[node] = machine(in, node, depth, r.Declarations) + } + return cv, nil +} + +// HoldingsOf is every seat an assigned module claims, held on the node it is assigned to, in node order: the holdings +// of an input that has none of record (the assigner's, mesh-delivery-assigner internal/assign input). +func HoldingsOf(shelf map[string]catalogue.Manifest, nodes []compose.Node) []catalogue.Held { + var out []catalogue.Held + sorted := slices.Clone(nodes) + sort.Slice(sorted, func(i, j int) bool { return sorted[i].Name < sorted[j].Name }) + for _, n := range sorted { + for _, mod := range n.Assigned { + for _, cl := range shelf[mod].Claims { + scope := cl.Scope + if seat, ok := catalogue.SeatNamed(cl.Name); ok && scope == "" { + scope = seat.Scope + } + out = append(out, catalogue.Held{Claim: cl.Name, Scope: scope, Node: n.Name, Module: mod}) + } + } + } + return out +} + +func machine(in compose.Input, node, depth string, declarations bool) Machine { + if depth == DepthResolve { + if _, err := compose.Resolve(in, node); err != nil { + return Machine{Reason: err.Error()} + } + return Machine{Composes: true} + } + d, problems, err := compose.Compose(in, node) + if err != nil { + return Machine{Reason: err.Error()} + } + m := Machine{Composes: len(problems) == 0, Problems: problems, Refusals: d.Refusals()} + if len(problems) > 0 { + m.Reason = "the node-engine's validator refuses the declaration" + } + if declarations { + m.Declared = pathsOf(d) + } + return m +} + +// pathsOf is every resource of a declaration that names a path, with its owner, group and mode as declared, sorted by +// path. +func pathsOf(d compose.Declared) []Path { + var out []Path + str := func(r map[string]any, key string) string { + if v, ok := r[key]; ok && v != nil { + return fmt.Sprint(v) + } + return "" + } + for _, r := range d.Resources { + p := str(r, "path") + if p == "" { + continue + } + out = append(out, Path{Path: p, Kind: str(r, "type"), Owner: str(r, "owner"), Group: str(r, "group"), + Mode: str(r, "mode"), Module: d.ModuleOf[str(r, "id")]}) + } + sort.Slice(out, func(i, j int) bool { + if out[i].Path != out[j].Path { + return out[i].Path < out[j].Path + } + return out[i].Kind < out[j].Kind + }) + return out +} diff --git a/go/composer/composer_test.go b/go/composer/composer_test.go new file mode 100644 index 0000000..175894a --- /dev/null +++ b/go/composer/composer_test.go @@ -0,0 +1,106 @@ +package composer + +import ( + "encoding/json" + "errors" + "reflect" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/compose" +) + +const demo = `{"module": "demo", "version": "1", "claims": [{"name": "node-files"}]}` + +// A manifest named by its digest goes on the wire as {"digest": ...}, and Fill puts the bytes the indexer keeps in its +// place; a raw manifest goes as itself. +func TestAManifestIsItsBytesOrItsDigest(t *testing.T) { + r := Request{V: Version, Cases: []Case{{Name: "change", Manifests: map[string]Manifest{ + "demo": {Digest: "d1"}, "raw": {Raw: json.RawMessage(`{"module":"raw","version":"1"}`)}}, + Kept: map[string]map[string]Manifest{"laptop": {"demo": {Digest: "d1"}}}}}} + wire, err := json.Marshal(r) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(wire), `"demo":{"digest":"d1"}`) || !strings.Contains(string(wire), `"raw":{"module":"raw","version":"1"}`) { + t.Fatalf("on the wire: %s", wire) + } + back, err := Decode(wire) + if err != nil { + t.Fatal(err) + } + if _, err := Run(back); !errors.Is(err, ErrUnfilled) { + t.Fatalf("a digest left unfilled runs: %v", err) + } + asked := 0 + if err := Fill(&back, func(d string) ([]byte, error) { + asked++ + if d != "d1" { + return nil, errors.New("no such digest") + } + return []byte(demo), nil + }); err != nil { + t.Fatal(err) + } + if asked != 2 || string(back.Cases[0].Manifests["demo"].Raw) != demo || string(back.Cases[0].Kept["laptop"]["demo"].Raw) != demo { + t.Fatalf("filled %d times: %+v", asked, back.Cases[0]) + } + bad := Request{V: Version, Cases: []Case{{Name: "c", Manifests: map[string]Manifest{"x": {Digest: "nope"}}}}} + if err := Fill(&bad, func(string) ([]byte, error) { return nil, errors.New("not there") }); err == nil || !strings.Contains(err.Error(), "nope") { + t.Fatalf("a digest the indexer does not hold: %v", err) + } +} + +// The input travels as JSON and comes back equal, so the command composes what its caller meant. +func TestTheInputTravelsAsJSON(t *testing.T) { + in := compose.Input{ + Nodes: []compose.Node{{Name: "anchor", Assigned: []string{"nats"}, Capabilities: map[string]bool{"systemd": true}, + Place: compose.Place{Endpoint: "anchor.example:51820", Site: "here", Hub: true, Address: "10.77.0.1"}, + Pins: map[string]catalogue.Chosen{"bus": {Node: "anchor", Module: "nats"}}, + Settings: map[string][]catalogue.Layer{"nats": {{Values: map[string]any{"port": float64(4222)}}}}}}, + Holdings: []catalogue.Held{{Claim: "mesh-broker", Scope: "mesh", Node: "anchor", Module: "nats"}}, + Issued: []compose.Issued{{Provision: "bus", Consumer: "laptop", ConsumerModule: "x", Provider: "anchor"}}, + } + raw, err := json.Marshal(Request{V: Version, Input: in}) + if err != nil { + t.Fatal(err) + } + back, err := Decode(raw) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(back.Input, in) { + t.Fatalf("the input came back as\n%+v\nwant\n%+v", back.Input, in) + } +} + +// The seats a node's assigned modules claim are held there when the request says the holdings come from the claims. +func TestHoldingsComeFromTheClaims(t *testing.T) { + m, err := catalogue.ParseManifest([]byte(demo)) + if err != nil { + t.Fatal(err) + } + got := HoldingsOf(map[string]catalogue.Manifest{"demo": m}, []compose.Node{{Name: "b", Assigned: []string{"demo"}}, {Name: "a", Assigned: []string{"demo"}}}) + if len(got) != 2 || got[0].Node != "a" || got[0].Claim != "node-files" || got[0].Scope != "node" { + t.Fatalf("holdings = %+v", got) + } +} + +// A request the command cannot run is refused, never answered as a verdict: another version, a case named twice, an +// unknown depth. +func TestARequestItCannotRunIsRefused(t *testing.T) { + if _, err := Decode([]byte(`{"v": 2}`)); err == nil { + t.Error("a request of version 2 is read") + } + if _, err := Run(Request{V: Version, Cases: []Case{{Name: "a"}, {Name: "a"}}}); err == nil { + t.Error("a case named twice runs") + } + if _, err := Run(Request{V: Version, Depth: "deep"}); err == nil { + t.Error("an unknown depth runs") + } + v, err := Run(Request{V: Version, Cases: []Case{{Name: "only-read", Manifests: map[string]Manifest{"demo": {Raw: json.RawMessage(demo)}}}}}) + if err != nil || len(v.Cases) != 1 || !v.Cases[0].Manifests["demo"].Accepted || v.Cases[0].Machines != nil || v.SDK == "" { + t.Fatalf("a request with no machines: %+v, %v", v, err) + } +}