Give mesh-sdk a composer command, so a program linking one mesh-sdk can have another commit's composition judged (issue 1621)
This commit is contained in:
@@ -0,0 +1,144 @@
|
||||
// mesh-compose is mesh-sdk's composer command (novox/hq ADR 0482 clause 4, issue 1621): it reads a composer.Request on
|
||||
// standard input, validates every manifest with this mesh-sdk's reader, composes the machines named in each case and
|
||||
// writes one composer.Verdict on standard output. Built at any commit of mesh-sdk (or at the mesh-sdk a module pins),
|
||||
// it answers what that commit's composition answers, so a program that links one mesh-sdk can have another's judged.
|
||||
//
|
||||
// mesh-compose < request.json > verdict.json
|
||||
// mesh-compose -catalogue <dir> [-node <name>]... [-depth resolve|compose] [-declarations] [< request.json]
|
||||
//
|
||||
// -catalogue reads <dir>/modules/*/module.json and a module.json at <dir>'s root as the first case's manifests (a
|
||||
// checkout or `git archive` of the catalogue repository at a ref: the command fetches nothing). -node adds a machine to
|
||||
// compose; -declarations answers each machine's declared paths with their owners and modes.
|
||||
//
|
||||
// It opens no network connection, bus, store or file beyond its standard input and the -catalogue directory, and reads
|
||||
// no environment. Exit codes: 0 the verdict is written (a machine that does not compose is a verdict, not an error);
|
||||
// 1 the request cannot be read or run, said on standard error; 2 the flags are wrong.
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/composer"
|
||||
)
|
||||
|
||||
func main() {
|
||||
os.Exit(run(os.Args[1:], os.Stdin, os.Stdout, os.Stderr))
|
||||
}
|
||||
|
||||
type nodes []string
|
||||
|
||||
func (n *nodes) String() string { return strings.Join(*n, ",") }
|
||||
func (n *nodes) Set(v string) error { *n = append(*n, v); return nil }
|
||||
|
||||
// run is the command: its arguments, standard input, output and error; it answers the exit code.
|
||||
func run(args []string, stdin io.Reader, stdout, stderr io.Writer) int {
|
||||
fs := flag.NewFlagSet("mesh-compose", flag.ContinueOnError)
|
||||
fs.SetOutput(stderr)
|
||||
catalogueDir := fs.String("catalogue", "", "a catalogue checkout: <dir>/modules/*/module.json are the first case's manifests")
|
||||
var machines nodes
|
||||
fs.Var(&machines, "node", "a machine to compose (repeatable)")
|
||||
depth := fs.String("depth", "", "resolve or compose (default: the request's, else compose)")
|
||||
declarations := fs.Bool("declarations", false, "answer each composed machine's declared paths, owners and modes")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return 2
|
||||
}
|
||||
if fs.NArg() > 0 {
|
||||
fmt.Fprintf(stderr, "mesh-compose: unexpected arguments %q\n", fs.Args())
|
||||
return 2
|
||||
}
|
||||
raw, err := io.ReadAll(stdin)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "mesh-compose: standard input cannot be read:", err)
|
||||
return 1
|
||||
}
|
||||
var req composer.Request
|
||||
if strings.TrimSpace(string(raw)) == "" {
|
||||
if *catalogueDir == "" {
|
||||
fmt.Fprintln(stderr, "mesh-compose: no request on standard input and no -catalogue")
|
||||
return 1
|
||||
}
|
||||
req = composer.Request{V: composer.Version}
|
||||
} else if req, err = composer.Decode(raw); err != nil {
|
||||
fmt.Fprintln(stderr, "mesh-compose:", err)
|
||||
return 1
|
||||
}
|
||||
if *catalogueDir != "" {
|
||||
read, err := readCatalogue(*catalogueDir)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "mesh-compose:", err)
|
||||
return 1
|
||||
}
|
||||
if len(req.Cases) == 0 {
|
||||
req.Cases = []composer.Case{{Name: "catalogue", Manifests: map[string]composer.Manifest{}}}
|
||||
}
|
||||
if req.Cases[0].Manifests == nil {
|
||||
req.Cases[0].Manifests = map[string]composer.Manifest{}
|
||||
}
|
||||
for name, m := range read {
|
||||
if _, given := req.Cases[0].Manifests[name]; !given {
|
||||
req.Cases[0].Manifests[name] = m
|
||||
}
|
||||
}
|
||||
}
|
||||
req.Machines = append(req.Machines, machines...)
|
||||
if *depth != "" {
|
||||
req.Depth = *depth
|
||||
}
|
||||
if *declarations {
|
||||
req.Declarations = true
|
||||
}
|
||||
v, err := composer.Run(req)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "mesh-compose:", err)
|
||||
return 1
|
||||
}
|
||||
enc := json.NewEncoder(stdout)
|
||||
if err := enc.Encode(v); err != nil {
|
||||
fmt.Fprintln(stderr, "mesh-compose: the verdict cannot be written:", err)
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// readCatalogue reads every module.json of a catalogue checkout by its module's name: the root's and each of
|
||||
// modules/*/. A manifest whose name cannot be read is kept under its directory's name, so the reader's refusal is said.
|
||||
func readCatalogue(dir string) (map[string]composer.Manifest, error) {
|
||||
paths, err := filepath.Glob(filepath.Join(dir, "modules", "*", "module.json"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "module.json")); err == nil {
|
||||
paths = append(paths, filepath.Join(dir, "module.json"))
|
||||
}
|
||||
if len(paths) == 0 {
|
||||
return nil, fmt.Errorf("the catalogue %s holds no module.json", dir)
|
||||
}
|
||||
sort.Strings(paths)
|
||||
out := map[string]composer.Manifest{}
|
||||
for _, p := range paths {
|
||||
raw, err := os.ReadFile(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var probe struct {
|
||||
Module string `json:"module"`
|
||||
}
|
||||
name := filepath.Base(filepath.Dir(p))
|
||||
if json.Unmarshal(raw, &probe) == nil && probe.Module != "" {
|
||||
name = probe.Module
|
||||
}
|
||||
if _, twice := out[name]; twice {
|
||||
return nil, errors.New("the catalogue " + dir + " defines " + name + " twice")
|
||||
}
|
||||
out[name] = composer.Manifest{Raw: raw}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,293 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"go/build"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
"git.novox.be/novox/mesh-sdk/go/composer"
|
||||
"git.novox.be/novox/mesh-sdk/go/internal/beside"
|
||||
)
|
||||
|
||||
// captured is every manifest of the catalogue captured in testdata/beside (internal/beside), raw, by module name.
|
||||
func captured(t *testing.T) map[string]composer.Manifest {
|
||||
t.Helper()
|
||||
read, err := readCatalogue(filepath.Dir(beside.Catalogue(t)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(read) < 100 {
|
||||
t.Fatalf("the captured catalogue holds %d manifests, fewer than the catalogue's", len(read))
|
||||
}
|
||||
return read
|
||||
}
|
||||
|
||||
// meshOf is a mesh of three machines over the captured catalogue, each assigned real modules, the input the assigner
|
||||
// composes with (nodes and their assignments, holdings from the claims): anchor runs the bus and the store, laptop a
|
||||
// desktop's modules, ace a module whose requirement is provided elsewhere.
|
||||
func meshOf() compose.Input {
|
||||
return compose.Input{Nodes: []compose.Node{
|
||||
{Name: "anchor", Assigned: []string{"nats", "postgres"}},
|
||||
{Name: "laptop", Assigned: []string{"sudo", "zsh"}},
|
||||
{Name: "ace", Assigned: []string{"mesh-reads", "nats"}},
|
||||
}}
|
||||
}
|
||||
|
||||
func request(t *testing.T, depth string) composer.Request {
|
||||
t.Helper()
|
||||
return composer.Request{V: composer.Version, Input: meshOf(), Depth: depth, HoldingsFromClaims: true,
|
||||
Machines: []string{"anchor", "laptop", "ace"},
|
||||
Cases: []composer.Case{{Name: "trunk", Manifests: captured(t)}}}
|
||||
}
|
||||
|
||||
func runJSON(t *testing.T, args []string, in []byte) (int, composer.Verdict, string) {
|
||||
t.Helper()
|
||||
var out, errOut bytes.Buffer
|
||||
code := run(args, bytes.NewReader(in), &out, &errOut)
|
||||
var v composer.Verdict
|
||||
if code == 0 {
|
||||
if err := json.Unmarshal(out.Bytes(), &v); err != nil {
|
||||
t.Fatalf("the verdict is not JSON: %v\n%s", err, out.String())
|
||||
}
|
||||
}
|
||||
return code, v, errOut.String()
|
||||
}
|
||||
|
||||
func encode(t *testing.T, r composer.Request) []byte {
|
||||
t.Helper()
|
||||
raw, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return raw
|
||||
}
|
||||
|
||||
// 1621/AC1: the command reads the plain input and the manifests on standard input, validates every manifest, composes
|
||||
// the machines named and writes one verdict: per machine composes or the reason, per manifest accepted or the field
|
||||
// refused.
|
||||
func TestTheCommandWritesOneVerdictPerManifestAndMachine(t *testing.T) {
|
||||
r := request(t, composer.DepthResolve)
|
||||
r.Cases[0].Manifests["broken"] = composer.Manifest{Raw: json.RawMessage(`{"module":"broken","version":"1","shares":["x"]}`)}
|
||||
code, v, stderr := runJSON(t, nil, encode(t, r))
|
||||
if code != 0 {
|
||||
t.Fatalf("exit %d: %s", code, stderr)
|
||||
}
|
||||
if len(v.Cases) != 1 || v.Cases[0].Name != "trunk" || v.SDK == "" {
|
||||
t.Fatalf("verdict = %+v", v)
|
||||
}
|
||||
c := v.Cases[0]
|
||||
if got := c.Manifests["broken"]; got.Accepted || !strings.Contains(got.Refused, `"shares"`) {
|
||||
t.Errorf("the manifest with an unknown field reads %+v, want refused naming the field", got)
|
||||
}
|
||||
if got := c.Manifests["nats"]; !got.Accepted {
|
||||
t.Errorf("nats reads %+v, want accepted", got)
|
||||
}
|
||||
if len(c.Machines) != 3 {
|
||||
t.Fatalf("machines = %v, want the three named", c.Machines)
|
||||
}
|
||||
for name, m := range c.Machines {
|
||||
if !m.Composes && m.Reason == "" {
|
||||
t.Errorf("%s does not compose and says no reason", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1621/AC3: over the captured catalogue and a mesh of three machines, the command's verdict equals, machine for
|
||||
// machine and manifest for manifest, what the linked reader, resolution and composition answer, at both depths.
|
||||
func TestTheCommandAnswersWhatTheLinkedCompositionAnswers(t *testing.T) {
|
||||
for _, depth := range []string{composer.DepthResolve, composer.DepthCompose} {
|
||||
r := request(t, depth)
|
||||
code, v, stderr := runJSON(t, nil, encode(t, r))
|
||||
if code != 0 {
|
||||
t.Fatalf("%s: exit %d: %s", depth, code, stderr)
|
||||
}
|
||||
in := meshOf()
|
||||
in.Catalogue = map[string]catalogue.Manifest{}
|
||||
for name, m := range r.Cases[0].Manifests {
|
||||
parsed, err := catalogue.ParseManifest(m.Raw)
|
||||
got := v.Cases[0].Manifests[name]
|
||||
if (err == nil) != got.Accepted || (err != nil && err.Error() != got.Refused) {
|
||||
t.Errorf("%s: the manifest of %s: linked %v, command %+v", depth, name, err, got)
|
||||
}
|
||||
if err == nil {
|
||||
in.Catalogue[name] = parsed
|
||||
}
|
||||
}
|
||||
in.Holdings = composer.HoldingsOf(in.Catalogue, in.Nodes)
|
||||
composed := 0
|
||||
for _, node := range r.Machines {
|
||||
got := v.Cases[0].Machines[node]
|
||||
var want composer.Machine
|
||||
if depth == composer.DepthResolve {
|
||||
if _, err := compose.Resolve(in, node); err != nil {
|
||||
want = composer.Machine{Reason: err.Error()}
|
||||
} else {
|
||||
want = composer.Machine{Composes: true}
|
||||
}
|
||||
} else {
|
||||
d, problems, err := compose.Compose(in, node)
|
||||
switch {
|
||||
case err != nil:
|
||||
want = composer.Machine{Reason: err.Error()}
|
||||
default:
|
||||
want = composer.Machine{Composes: len(problems) == 0, Problems: problems, Refusals: d.Refusals()}
|
||||
if len(problems) > 0 {
|
||||
want.Reason = "the node-engine's validator refuses the declaration"
|
||||
}
|
||||
}
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("%s: %s: command %+v, linked %+v", depth, node, got, want)
|
||||
}
|
||||
if got.Composes {
|
||||
composed++
|
||||
} else {
|
||||
t.Logf("%s: %s does not compose: %s %v", depth, node, got.Reason, got.Problems)
|
||||
}
|
||||
}
|
||||
t.Logf("%s: %d of %d machines compose", depth, composed, len(r.Machines))
|
||||
}
|
||||
}
|
||||
|
||||
// 1621 note 6: a person composes a catalogue checkout (repository@ref checked out or archived: the command fetches
|
||||
// nothing) for a node and reads its declaration: each path with its kind, owner and mode.
|
||||
func TestACatalogueCheckoutIsComposedForANode(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(dir, "modules", "demo"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
manifest := `{"module": "demo", "version": "1", "resources": [{"id": "state", "type": "directory", ` +
|
||||
`"path": "/var/lib/demo", "mode": "0750", "owner": "root"}]}`
|
||||
if err := os.WriteFile(filepath.Join(dir, "modules", "demo", "module.json"), []byte(manifest), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
in := encode(t, composer.Request{V: composer.Version, Input: compose.Input{Nodes: []compose.Node{{Name: "laptop", Assigned: []string{"demo"}}}}})
|
||||
code, v, stderr := runJSON(t, []string{"-catalogue", dir, "-node", "laptop", "-declarations"}, in)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit %d: %s", code, stderr)
|
||||
}
|
||||
m := v.Cases[0].Machines["laptop"]
|
||||
want := []composer.Path{{Path: "/var/lib/demo", Kind: "directory", Owner: "root", Mode: "0750", Module: ""}}
|
||||
if !m.Composes || len(m.Declared) != 1 || m.Declared[0].Path != want[0].Path || m.Declared[0].Mode != "0750" ||
|
||||
m.Declared[0].Owner != "root" || m.Declared[0].Kind != "directory" {
|
||||
t.Fatalf("laptop = %+v, want it composed declaring %+v", m, want)
|
||||
}
|
||||
}
|
||||
|
||||
// The exit codes: 1 for a request that cannot be read or run, 2 for wrong flags, each said on standard error.
|
||||
func TestTheExitCodesSayWhatFailed(t *testing.T) {
|
||||
unfilled := encode(t, composer.Request{V: composer.Version, Cases: []composer.Case{{Name: "c", Manifests: map[string]composer.Manifest{"x": {Digest: "abc"}}}}})
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
args []string
|
||||
in string
|
||||
code int
|
||||
says string
|
||||
}{
|
||||
{"not JSON", nil, "{", 1, "not JSON"},
|
||||
{"another version", nil, `{"v":2}`, 1, "version 2"},
|
||||
{"a digest left unfilled", nil, string(unfilled), 1, "digest"},
|
||||
{"an unknown depth", nil, `{"v":1,"depth":"deep"}`, 1, "depth"},
|
||||
{"nothing to read", nil, "", 1, "no request"},
|
||||
{"a wrong flag", []string{"-nope"}, "", 2, "nope"},
|
||||
{"an argument", []string{"extra"}, "", 2, "unexpected"},
|
||||
} {
|
||||
var out, errOut bytes.Buffer
|
||||
if got := run(c.args, strings.NewReader(c.in), &out, &errOut); got != c.code || !strings.Contains(errOut.String(), c.says) {
|
||||
t.Errorf("%s: exit %d saying %q, want %d saying %q", c.name, got, errOut.String(), c.code, c.says)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sdkPackages is every package of this module the command imports, directly or not, with the packages each imports.
|
||||
func sdkPackages(t *testing.T) map[string][]string {
|
||||
t.Helper()
|
||||
_, file, _, _ := runtime.Caller(0)
|
||||
root := filepath.Join(filepath.Dir(file), "..", "..")
|
||||
const module = "git.novox.be/novox/mesh-sdk/go"
|
||||
out := map[string][]string{}
|
||||
var walk func(path string)
|
||||
walk = func(path string) {
|
||||
if _, done := out[path]; done {
|
||||
return
|
||||
}
|
||||
rel := strings.TrimPrefix(strings.TrimPrefix(path, module), "/")
|
||||
pkg, err := build.Default.ImportDir(filepath.Join(root, rel), 0)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", path, err)
|
||||
}
|
||||
out[path] = pkg.Imports
|
||||
for _, imp := range pkg.Imports {
|
||||
if strings.HasPrefix(imp, module+"/") {
|
||||
walk(imp)
|
||||
}
|
||||
}
|
||||
}
|
||||
walk(composer.Package)
|
||||
return out
|
||||
}
|
||||
|
||||
// 1621/AC2 and AC4: the command imports, through every package of mesh-sdk it reaches, only the standard library and
|
||||
// mesh-sdk's own packages (so `go build` from mesh-sdk's module needs nothing beyond its lock), and none that reaches
|
||||
// a bus, a store, a network service or another process.
|
||||
func TestTheCommandImportsNoBusStoreNetworkOrProcess(t *testing.T) {
|
||||
forbidden := []string{"net/http", "net/rpc", "net/smtp", "os/exec", "database/sql", "plugin", "syscall"}
|
||||
for path, imports := range sdkPackages(t) {
|
||||
for _, imp := range imports {
|
||||
std := !strings.Contains(strings.SplitN(imp, "/", 2)[0], ".")
|
||||
if !std && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-sdk/go/") {
|
||||
t.Errorf("%s imports %s, which is neither the standard library nor mesh-sdk", path, imp)
|
||||
}
|
||||
for _, f := range forbidden {
|
||||
if imp == f {
|
||||
t.Errorf("%s imports %s", path, imp)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1621/AC2: run with an empty network namespace, the command still writes its verdict: it needs no network. The test
|
||||
// runs its own binary again under `unshare --net` as an unprivileged user's namespace; where the kernel or the sandbox
|
||||
// allows no new namespace (a build module's run step is already one with no network), it says so and skips.
|
||||
func TestTheCommandRunsWithNoNetwork(t *testing.T) {
|
||||
if os.Getenv("MESH_COMPOSE_HELPER") == "1" {
|
||||
os.Exit(run(nil, os.Stdin, os.Stdout, os.Stderr))
|
||||
}
|
||||
unshare, err := exec.LookPath("unshare")
|
||||
if err != nil {
|
||||
t.Skip("no unshare on this machine, so no empty network namespace can be made here")
|
||||
}
|
||||
if out, err := exec.Command(unshare, "--net", "--map-root-user", "true").CombinedOutput(); err != nil {
|
||||
t.Skipf("this machine makes no unprivileged network namespace (%v: %s); a run step's own has no network", err, out)
|
||||
}
|
||||
r := request(t, composer.DepthResolve)
|
||||
cmd := exec.Command(unshare, "--net", "--map-root-user", os.Args[0], "-test.run=^TestTheCommandRunsWithNoNetwork$")
|
||||
cmd.Env = []string{"MESH_COMPOSE_HELPER=1"}
|
||||
cmd.Stdin = bytes.NewReader(encode(t, r))
|
||||
var out, errOut bytes.Buffer
|
||||
cmd.Stdout, cmd.Stderr = &out, &errOut
|
||||
if err := cmd.Run(); err != nil {
|
||||
t.Fatalf("the command in an empty network namespace: %v: %s", err, errOut.String())
|
||||
}
|
||||
var v composer.Verdict
|
||||
if err := json.Unmarshal(out.Bytes(), &v); err != nil || len(v.Cases) != 1 || len(v.Cases[0].Machines) != 3 {
|
||||
t.Fatalf("no verdict from the command with no network: %v\n%s", err, out.String())
|
||||
}
|
||||
names := make([]string, 0, len(v.Cases[0].Machines))
|
||||
for n := range v.Cases[0].Machines {
|
||||
names = append(names, n)
|
||||
}
|
||||
sort.Strings(names)
|
||||
t.Logf("composed with no network: %v", names)
|
||||
}
|
||||
@@ -0,0 +1,332 @@
|
||||
// Package composer is mesh-sdk's composition and validator as one function over a plain request (novox/hq ADR 0482
|
||||
// clause 4, issue 1621): what the command mesh-compose runs, so a program that links any one mesh-sdk can have the
|
||||
// composition of another commit judged by building that commit's command and reading its verdict.
|
||||
//
|
||||
// **It reads its request and nothing else.** No bus, store, network, file or environment: the manifests come inside the
|
||||
// request (raw, as their repository holds them), the rest of the mesh is a compose.Input, and the answer is a value.
|
||||
// A manifest named only by its digest in the delivery indexer's bucket is filled before the run by Fill, by whoever
|
||||
// holds the bus (the Go build module's compose job); Run refuses a digest left unfilled.
|
||||
package composer
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
"git.novox.be/novox/mesh-sdk/go/sdkversion"
|
||||
)
|
||||
|
||||
// Version is the version of a Request and of a Verdict; a reader refuses another.
|
||||
const Version = 1
|
||||
|
||||
// Package is the import path of the command, which a compose job builds at a commit (in mesh-sdk's own module, or in
|
||||
// a module that requires mesh-sdk, at the version it pins).
|
||||
const Package = "git.novox.be/novox/mesh-sdk/go/cmd/mesh-compose"
|
||||
|
||||
// The depths of a composition.
|
||||
const (
|
||||
// DepthResolve resolves each machine's set over the rest of the mesh (compose.Resolve): what the assigner judges
|
||||
// while the facts snapshot is not on the bus (issue 1461).
|
||||
DepthResolve = "resolve"
|
||||
// DepthCompose composes each machine's whole declaration and runs the node-engine's validator over it
|
||||
// (compose.Compose).
|
||||
DepthCompose = "compose"
|
||||
)
|
||||
|
||||
// Request is everything one run composes: the rest of the mesh, one or more cases of manifests composed over it, and
|
||||
// the machines composed in each case.
|
||||
type Request struct {
|
||||
V int `json:"v"`
|
||||
// Input is the composition's input without its catalogue (and without each node's kept manifests): the cases give
|
||||
// those. A Catalogue or Kept it holds is ignored.
|
||||
Input compose.Input `json:"input"`
|
||||
// Cases are composed one after the other over Input, each with its own manifests; their names are unique.
|
||||
Cases []Case `json:"cases"`
|
||||
// Machines are the nodes composed in every case; none means the manifests are only read.
|
||||
Machines []string `json:"machines,omitempty"`
|
||||
// Depth is DepthResolve or DepthCompose; empty is DepthCompose.
|
||||
Depth string `json:"depth,omitempty"`
|
||||
// HoldingsFromClaims adds to Input's holdings every seat an assigned module claims, held on its node (the
|
||||
// assigner's input, which has no holdings of record).
|
||||
HoldingsFromClaims bool `json:"holdings_from_claims,omitempty"`
|
||||
// Declarations asks each composed machine's declared paths (DepthCompose only).
|
||||
Declarations bool `json:"declarations,omitempty"`
|
||||
}
|
||||
|
||||
// Case is one catalogue composed: every module's manifest by name, and per node the manifests kept at the build it runs.
|
||||
type Case struct {
|
||||
Name string `json:"name"`
|
||||
Manifests map[string]Manifest `json:"manifests"`
|
||||
Kept map[string]map[string]Manifest `json:"kept,omitempty"`
|
||||
}
|
||||
|
||||
// Manifest is a module's manifest as its repository holds it (Raw, the bytes), or the digest the delivery indexer keeps
|
||||
// it under (Digest), which Fill replaces by the bytes. On the wire it is the manifest itself, or {"digest": "..."}.
|
||||
type Manifest struct {
|
||||
Raw json.RawMessage
|
||||
Digest string
|
||||
}
|
||||
|
||||
// MarshalJSON writes the raw manifest, or {"digest": ...} when only the digest is known.
|
||||
func (m Manifest) MarshalJSON() ([]byte, error) {
|
||||
if len(m.Raw) > 0 {
|
||||
return m.Raw, nil
|
||||
}
|
||||
return json.Marshal(map[string]string{"digest": m.Digest})
|
||||
}
|
||||
|
||||
// UnmarshalJSON reads {"digest": "..."} (an object with that one key) as a digest, and any other value as the raw
|
||||
// manifest.
|
||||
func (m *Manifest) UnmarshalJSON(raw []byte) error {
|
||||
var probe map[string]json.RawMessage
|
||||
if json.Unmarshal(raw, &probe) == nil && len(probe) == 1 {
|
||||
if d, ok := probe["digest"]; ok {
|
||||
var digest string
|
||||
if err := json.Unmarshal(d, &digest); err != nil {
|
||||
return fmt.Errorf("a manifest's digest is not a string: %w", err)
|
||||
}
|
||||
*m = Manifest{Digest: digest}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
*m = Manifest{Raw: slices.Clone(raw)}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Verdict is what one run answers: the mesh-sdk the command was built with, and each case's verdict in the order asked.
|
||||
type Verdict struct {
|
||||
V int `json:"v"`
|
||||
SDK string `json:"sdk"`
|
||||
Cases []CaseVerdict `json:"cases"`
|
||||
}
|
||||
|
||||
// CaseVerdict is one case's: every manifest read or refused, every machine composed or not.
|
||||
type CaseVerdict struct {
|
||||
Name string `json:"name"`
|
||||
Manifests map[string]ManifestRead `json:"manifests"`
|
||||
Machines map[string]Machine `json:"machines,omitempty"`
|
||||
}
|
||||
|
||||
// ManifestRead is one manifest as this mesh-sdk reads it: accepted, or refused with the reader's words.
|
||||
type ManifestRead struct {
|
||||
Accepted bool `json:"accepted"`
|
||||
Refused string `json:"refused,omitempty"`
|
||||
}
|
||||
|
||||
// Machine is one machine composed: whether it composes (resolves, at DepthResolve), why not, the node-engine
|
||||
// validator's problems and the composition's refusals (DepthCompose), and its declared paths when asked.
|
||||
type Machine struct {
|
||||
Composes bool `json:"composes"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
Problems []string `json:"problems,omitempty"`
|
||||
Refusals []string `json:"refusals,omitempty"`
|
||||
Declared []Path `json:"declared,omitempty"`
|
||||
}
|
||||
|
||||
// Path is one path a declaration puts on the machine, with its kind (the resource's type), owner, group and mode as
|
||||
// declared, and the module it is of.
|
||||
type Path struct {
|
||||
Path string `json:"path"`
|
||||
Kind string `json:"kind"`
|
||||
Owner string `json:"owner,omitempty"`
|
||||
Group string `json:"group,omitempty"`
|
||||
Mode string `json:"mode,omitempty"`
|
||||
Module string `json:"module,omitempty"`
|
||||
}
|
||||
|
||||
// ErrUnfilled is a manifest named by a digest that nobody replaced by its bytes.
|
||||
var ErrUnfilled = errors.New("composer: a manifest is named by its digest and was not filled")
|
||||
|
||||
// Decode reads a request; it refuses one that is not JSON or of another version.
|
||||
func Decode(data []byte) (Request, error) {
|
||||
var r Request
|
||||
if err := json.Unmarshal(data, &r); err != nil {
|
||||
return Request{}, fmt.Errorf("composer: the request is not JSON of its shape: %w", err)
|
||||
}
|
||||
if r.V != Version {
|
||||
return Request{}, fmt.Errorf("composer: a request of version %d, this composer reads %d", r.V, Version)
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// Fill replaces every manifest named by its digest by the bytes get answers for that digest; a digest get cannot answer
|
||||
// stops it, named.
|
||||
func Fill(r *Request, get func(digest string) ([]byte, error)) error {
|
||||
fill := func(m map[string]Manifest) error {
|
||||
for name, man := range m {
|
||||
if len(man.Raw) > 0 {
|
||||
continue
|
||||
}
|
||||
raw, err := get(man.Digest)
|
||||
if err != nil {
|
||||
return fmt.Errorf("composer: the manifest of %s (digest %s): %w", name, man.Digest, err)
|
||||
}
|
||||
m[name] = Manifest{Raw: raw}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
for _, c := range r.Cases {
|
||||
if err := fill(c.Manifests); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, kept := range c.Kept {
|
||||
if err := fill(kept); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Run composes every case of r and answers the verdict, its SDK this binary's mesh-sdk. An error is a request it
|
||||
// cannot run (an unknown depth, a case named twice, a manifest left as a digest); a machine that does not compose is a
|
||||
// verdict.
|
||||
func Run(r Request) (Verdict, error) {
|
||||
depth := r.Depth
|
||||
if depth == "" {
|
||||
depth = DepthCompose
|
||||
}
|
||||
if depth != DepthResolve && depth != DepthCompose {
|
||||
return Verdict{}, fmt.Errorf("composer: the depth %q is not resolve or compose", r.Depth)
|
||||
}
|
||||
v := Verdict{V: Version, SDK: sdkversion.Version()}
|
||||
seen := map[string]bool{}
|
||||
for _, c := range r.Cases {
|
||||
if seen[c.Name] {
|
||||
return Verdict{}, fmt.Errorf("composer: the case %q is named twice", c.Name)
|
||||
}
|
||||
seen[c.Name] = true
|
||||
cv, err := runCase(r, c, depth)
|
||||
if err != nil {
|
||||
return Verdict{}, err
|
||||
}
|
||||
v.Cases = append(v.Cases, cv)
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
|
||||
func parse(m map[string]Manifest, reads map[string]ManifestRead) (map[string]catalogue.Manifest, error) {
|
||||
out := map[string]catalogue.Manifest{}
|
||||
for name, man := range m {
|
||||
if len(man.Raw) == 0 {
|
||||
return nil, fmt.Errorf("%w: %s (digest %s)", ErrUnfilled, name, man.Digest)
|
||||
}
|
||||
parsed, err := catalogue.ParseManifest(man.Raw)
|
||||
if reads != nil {
|
||||
if err != nil {
|
||||
reads[name] = ManifestRead{Refused: err.Error()}
|
||||
} else {
|
||||
reads[name] = ManifestRead{Accepted: true}
|
||||
}
|
||||
}
|
||||
if err == nil {
|
||||
out[name] = parsed
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func runCase(r Request, c Case, depth string) (CaseVerdict, error) {
|
||||
cv := CaseVerdict{Name: c.Name, Manifests: map[string]ManifestRead{}}
|
||||
shelf, err := parse(c.Manifests, cv.Manifests)
|
||||
if err != nil {
|
||||
return CaseVerdict{}, err
|
||||
}
|
||||
in := r.Input
|
||||
in.Catalogue = shelf
|
||||
in.Nodes = slices.Clone(in.Nodes)
|
||||
in.Holdings = slices.Clone(in.Holdings)
|
||||
for i := range in.Nodes {
|
||||
in.Nodes[i].Kept = nil
|
||||
if kept := c.Kept[in.Nodes[i].Name]; len(kept) > 0 {
|
||||
k, err := parse(kept, nil)
|
||||
if err != nil {
|
||||
return CaseVerdict{}, err
|
||||
}
|
||||
in.Nodes[i].Kept = k
|
||||
}
|
||||
}
|
||||
if r.HoldingsFromClaims {
|
||||
in.Holdings = append(in.Holdings, HoldingsOf(shelf, in.Nodes)...)
|
||||
}
|
||||
if len(r.Machines) == 0 {
|
||||
return cv, nil
|
||||
}
|
||||
cv.Machines = map[string]Machine{}
|
||||
for _, node := range r.Machines {
|
||||
cv.Machines[node] = machine(in, node, depth, r.Declarations)
|
||||
}
|
||||
return cv, nil
|
||||
}
|
||||
|
||||
// HoldingsOf is every seat an assigned module claims, held on the node it is assigned to, in node order: the holdings
|
||||
// of an input that has none of record (the assigner's, mesh-delivery-assigner internal/assign input).
|
||||
func HoldingsOf(shelf map[string]catalogue.Manifest, nodes []compose.Node) []catalogue.Held {
|
||||
var out []catalogue.Held
|
||||
sorted := slices.Clone(nodes)
|
||||
sort.Slice(sorted, func(i, j int) bool { return sorted[i].Name < sorted[j].Name })
|
||||
for _, n := range sorted {
|
||||
for _, mod := range n.Assigned {
|
||||
for _, cl := range shelf[mod].Claims {
|
||||
scope := cl.Scope
|
||||
if seat, ok := catalogue.SeatNamed(cl.Name); ok && scope == "" {
|
||||
scope = seat.Scope
|
||||
}
|
||||
out = append(out, catalogue.Held{Claim: cl.Name, Scope: scope, Node: n.Name, Module: mod})
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func machine(in compose.Input, node, depth string, declarations bool) Machine {
|
||||
if depth == DepthResolve {
|
||||
if _, err := compose.Resolve(in, node); err != nil {
|
||||
return Machine{Reason: err.Error()}
|
||||
}
|
||||
return Machine{Composes: true}
|
||||
}
|
||||
d, problems, err := compose.Compose(in, node)
|
||||
if err != nil {
|
||||
return Machine{Reason: err.Error()}
|
||||
}
|
||||
m := Machine{Composes: len(problems) == 0, Problems: problems, Refusals: d.Refusals()}
|
||||
if len(problems) > 0 {
|
||||
m.Reason = "the node-engine's validator refuses the declaration"
|
||||
}
|
||||
if declarations {
|
||||
m.Declared = pathsOf(d)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// pathsOf is every resource of a declaration that names a path, with its owner, group and mode as declared, sorted by
|
||||
// path.
|
||||
func pathsOf(d compose.Declared) []Path {
|
||||
var out []Path
|
||||
str := func(r map[string]any, key string) string {
|
||||
if v, ok := r[key]; ok && v != nil {
|
||||
return fmt.Sprint(v)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
for _, r := range d.Resources {
|
||||
p := str(r, "path")
|
||||
if p == "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, Path{Path: p, Kind: str(r, "type"), Owner: str(r, "owner"), Group: str(r, "group"),
|
||||
Mode: str(r, "mode"), Module: d.ModuleOf[str(r, "id")]})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].Path != out[j].Path {
|
||||
return out[i].Path < out[j].Path
|
||||
}
|
||||
return out[i].Kind < out[j].Kind
|
||||
})
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package composer
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
)
|
||||
|
||||
const demo = `{"module": "demo", "version": "1", "claims": [{"name": "node-files"}]}`
|
||||
|
||||
// A manifest named by its digest goes on the wire as {"digest": ...}, and Fill puts the bytes the indexer keeps in its
|
||||
// place; a raw manifest goes as itself.
|
||||
func TestAManifestIsItsBytesOrItsDigest(t *testing.T) {
|
||||
r := Request{V: Version, Cases: []Case{{Name: "change", Manifests: map[string]Manifest{
|
||||
"demo": {Digest: "d1"}, "raw": {Raw: json.RawMessage(`{"module":"raw","version":"1"}`)}},
|
||||
Kept: map[string]map[string]Manifest{"laptop": {"demo": {Digest: "d1"}}}}}}
|
||||
wire, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(wire), `"demo":{"digest":"d1"}`) || !strings.Contains(string(wire), `"raw":{"module":"raw","version":"1"}`) {
|
||||
t.Fatalf("on the wire: %s", wire)
|
||||
}
|
||||
back, err := Decode(wire)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := Run(back); !errors.Is(err, ErrUnfilled) {
|
||||
t.Fatalf("a digest left unfilled runs: %v", err)
|
||||
}
|
||||
asked := 0
|
||||
if err := Fill(&back, func(d string) ([]byte, error) {
|
||||
asked++
|
||||
if d != "d1" {
|
||||
return nil, errors.New("no such digest")
|
||||
}
|
||||
return []byte(demo), nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if asked != 2 || string(back.Cases[0].Manifests["demo"].Raw) != demo || string(back.Cases[0].Kept["laptop"]["demo"].Raw) != demo {
|
||||
t.Fatalf("filled %d times: %+v", asked, back.Cases[0])
|
||||
}
|
||||
bad := Request{V: Version, Cases: []Case{{Name: "c", Manifests: map[string]Manifest{"x": {Digest: "nope"}}}}}
|
||||
if err := Fill(&bad, func(string) ([]byte, error) { return nil, errors.New("not there") }); err == nil || !strings.Contains(err.Error(), "nope") {
|
||||
t.Fatalf("a digest the indexer does not hold: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The input travels as JSON and comes back equal, so the command composes what its caller meant.
|
||||
func TestTheInputTravelsAsJSON(t *testing.T) {
|
||||
in := compose.Input{
|
||||
Nodes: []compose.Node{{Name: "anchor", Assigned: []string{"nats"}, Capabilities: map[string]bool{"systemd": true},
|
||||
Place: compose.Place{Endpoint: "anchor.example:51820", Site: "here", Hub: true, Address: "10.77.0.1"},
|
||||
Pins: map[string]catalogue.Chosen{"bus": {Node: "anchor", Module: "nats"}},
|
||||
Settings: map[string][]catalogue.Layer{"nats": {{Values: map[string]any{"port": float64(4222)}}}}}},
|
||||
Holdings: []catalogue.Held{{Claim: "mesh-broker", Scope: "mesh", Node: "anchor", Module: "nats"}},
|
||||
Issued: []compose.Issued{{Provision: "bus", Consumer: "laptop", ConsumerModule: "x", Provider: "anchor"}},
|
||||
}
|
||||
raw, err := json.Marshal(Request{V: Version, Input: in})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
back, err := Decode(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(back.Input, in) {
|
||||
t.Fatalf("the input came back as\n%+v\nwant\n%+v", back.Input, in)
|
||||
}
|
||||
}
|
||||
|
||||
// The seats a node's assigned modules claim are held there when the request says the holdings come from the claims.
|
||||
func TestHoldingsComeFromTheClaims(t *testing.T) {
|
||||
m, err := catalogue.ParseManifest([]byte(demo))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := HoldingsOf(map[string]catalogue.Manifest{"demo": m}, []compose.Node{{Name: "b", Assigned: []string{"demo"}}, {Name: "a", Assigned: []string{"demo"}}})
|
||||
if len(got) != 2 || got[0].Node != "a" || got[0].Claim != "node-files" || got[0].Scope != "node" {
|
||||
t.Fatalf("holdings = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A request the command cannot run is refused, never answered as a verdict: another version, a case named twice, an
|
||||
// unknown depth.
|
||||
func TestARequestItCannotRunIsRefused(t *testing.T) {
|
||||
if _, err := Decode([]byte(`{"v": 2}`)); err == nil {
|
||||
t.Error("a request of version 2 is read")
|
||||
}
|
||||
if _, err := Run(Request{V: Version, Cases: []Case{{Name: "a"}, {Name: "a"}}}); err == nil {
|
||||
t.Error("a case named twice runs")
|
||||
}
|
||||
if _, err := Run(Request{V: Version, Depth: "deep"}); err == nil {
|
||||
t.Error("an unknown depth runs")
|
||||
}
|
||||
v, err := Run(Request{V: Version, Cases: []Case{{Name: "only-read", Manifests: map[string]Manifest{"demo": {Raw: json.RawMessage(demo)}}}}})
|
||||
if err != nil || len(v.Cases) != 1 || !v.Cases[0].Manifests["demo"].Accepted || v.Cases[0].Machines != nil || v.SDK == "" {
|
||||
t.Fatalf("a request with no machines: %+v, %v", v, err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user