diff --git a/go/buildstep/pull_test.go b/go/buildstep/pull_test.go index 6d0b326..ddfd33a 100644 --- a/go/buildstep/pull_test.go +++ b/go/buildstep/pull_test.go @@ -127,11 +127,8 @@ func TestPullHoldsAtMostItsCapacity(t *testing.T) { } release <- struct{}{} waitFor(t, "the third taken", func() bool { return src.Waiting(delivery.AskDelivery) == 0 }) - mu.Lock() - if started != 3 { - t.Errorf("started %d, want 3", started) - } - mu.Unlock() + // Taken from the source comes a moment before its run starts: wait for the start, not only the take. + waitFor(t, "the third started", func() bool { mu.Lock(); defer mu.Unlock(); return started == 3 }) close(release) waitFor(t, "all acked", func() bool { a, _, _ := src.Fates(); return len(a) == 3 }) } diff --git a/go/catalogue/reads-outside b/go/catalogue/reads-outside new file mode 100644 index 0000000..976b0f5 --- /dev/null +++ b/go/catalogue/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the repository's root files or other directories): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/go/catalogue/verbs.go b/go/catalogue/verbs.go index 2218d95..11f0b3e 100644 --- a/go/catalogue/verbs.go +++ b/go/catalogue/verbs.go @@ -81,7 +81,7 @@ var RemovedControllerVerbs = map[string]string{ "plans": "the delivery view (nox-delivery): each machine's want and have, read from the assigner's wants and the " + "node-engine's haves", "plan": "the node's compositions: mesh-controller_composed holds each module's digest and judgement per node, and " + - "mesh-compose prints one module's composition for one node", + "mesh-hand-compose prints one module's composition for one node", "push": "a want: the assigner writes want.. by its rules, and by hand the operator's credential writes " + "it, naming a kept composition (ADR 0539 clause 10)", "deliver": "nothing to start: a merged change reaches each machine by its want", diff --git a/go/cmd/mesh-compose/main.go b/go/cmd/mesh-compose/main.go index 0a849d8..3168bde 100644 --- a/go/cmd/mesh-compose/main.go +++ b/go/cmd/mesh-compose/main.go @@ -1,209 +1,163 @@ -// mesh-compose composes one module for one node by hand, from the inputs the controller keeps, and signs it with the -// operator's key, for the hand path of novox/hq ADR 0539 clause 10: a fingerprint the controller never composed (a -// hand build), while the controller cannot compose its own fix. It composes with mesh-sdk's composition, the one the -// controller writes with (compose.Compositions), and prints the composition, its key and its digest; only with -write -// does it write them, with the operator's credential: the composition into the node's bucket composed-, each -// file over 64 KiB into the node's object bucket, and the digest into mesh-controller_composed. The want is the -// operator's next act, naming that digest; each act is recorded in the hand-act log. +// mesh-compose is mesh-sdk's composer command (novox/hq ADR 0482 clause 4, issue 1621): it reads a composer.Request on +// standard input, validates every manifest with this mesh-sdk's reader, composes the machines named in each case and +// writes one composer.Verdict on standard output. Built at any commit of mesh-sdk (or at the mesh-sdk a module pins), +// it answers what that commit's composition answers, so a program that links one mesh-sdk can have another's judged. // -// mesh-compose -input -node -module -fp -key -// [-write -nats -creds ] +// mesh-compose < request.json > verdict.json +// mesh-compose -request request.json > verdict.json +// mesh-compose -catalogue [-node ]... [-depth resolve|compose] [-declarations] [< request.json] // -// The inputs are a compose.Input as JSON. The key is the operator's ed25519 private key, PKCS#8 in PEM; the -// node-engine trusts it beside the controller's (ADR 0539 clause 3). +// -catalogue reads /modules/*/module.json and a module.json at 's root as the first case's manifests (a +// checkout or `git archive` of the catalogue repository at a ref: the command fetches nothing). -node adds a machine to +// compose; -declarations answers each machine's declared paths with their owners and modes. +// +// It opens no network connection, bus, store or file beyond its standard input, the -request file and the -catalogue +// directory, and reads +// no environment. Exit codes: 0 the verdict is written (a machine that does not compose is a verdict, not an error); +// 1 the request cannot be read or run, said on standard error; 2 the flags are wrong. package main import ( - "context" - "crypto/ed25519" - "crypto/x509" "encoding/json" - "encoding/pem" "errors" "flag" "fmt" "io" "os" - "time" + "path/filepath" + "sort" + "strings" - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" - - "git.novox.be/novox/mesh-sdk/go/compose" - "git.novox.be/novox/mesh-sdk/go/declaration" - "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/composer" ) func main() { - if err := run(os.Args[1:], os.Stdout); err != nil { - fmt.Fprintln(os.Stderr, "mesh-compose:", err) - os.Exit(1) - } + os.Exit(run(os.Args[1:], os.Stdin, os.Stdout, os.Stderr)) } -// printed is what mesh-compose prints: where the composition goes, its digest, the composition, and whether it was -// written. -type printed struct { - Bucket string `json:"bucket"` - Key string `json:"key"` - Digest string `json:"digest"` - Composition json.RawMessage `json:"composition"` - Objects []string `json:"objects,omitempty"` - Written bool `json:"written"` - Next string `json:"next"` -} +type nodes []string -func run(args []string, stdout io.Writer) error { +func (n *nodes) String() string { return strings.Join(*n, ",") } +func (n *nodes) Set(v string) error { *n = append(*n, v); return nil } + +// run is the command: its arguments, standard input, output and error; it answers the exit code. +func run(args []string, stdin io.Reader, stdout, stderr io.Writer) int { fs := flag.NewFlagSet("mesh-compose", flag.ContinueOnError) - input := fs.String("input", "", "the controller's kept inputs, a compose.Input as JSON") - node := fs.String("node", "", "the node to compose for") - module := fs.String("module", "", "the module to compose") - fp := fs.String("fp", "", "the module's fingerprint, src2: and 64 hex digits") - keyFile := fs.String("key", "", "the operator's ed25519 private key, PKCS#8 in PEM") - write := fs.Bool("write", false, "write the composition, its files and its digest, with the operator's credential") - url := fs.String("nats", nats.DefaultURL, "with -write: the bus") - creds := fs.String("creds", "", "with -write: the operator's credential file") + fs.SetOutput(stderr) + catalogueDir := fs.String("catalogue", "", "a catalogue checkout: /modules/*/module.json are the first case's manifests") + var machines nodes + fs.Var(&machines, "node", "a machine to compose (repeatable)") + depth := fs.String("depth", "", "resolve or compose (default: the request's, else compose)") + declarations := fs.Bool("declarations", false, "answer each composed machine's declared paths, owners and modes") + requestFile := fs.String("request", "", "read the request from this file instead of standard input (a run step that hands no standard input)") if err := fs.Parse(args); err != nil { - return err + return 2 } - switch { - case *input == "" || *node == "" || *module == "" || *fp == "": - return errors.New("-input, -node, -module and -fp are each required") - case *keyFile == "": - return errors.New("-key is required: a composition made by hand is signed by the operator's key, and the node-engine applies no other") - case *write && *creds == "": - return errors.New("-write needs -creds: the operator's credential is the only hand writer of a composition") + if fs.NArg() > 0 { + fmt.Fprintf(stderr, "mesh-compose: unexpected arguments %q\n", fs.Args()) + return 2 + } + var raw []byte + var err error + if *requestFile != "" { + raw, err = os.ReadFile(*requestFile) + } else { + raw, err = io.ReadAll(stdin) } - key, err := readKey(*keyFile) if err != nil { - return err + fmt.Fprintln(stderr, "mesh-compose: the request cannot be read:", err) + return 1 } - key2, err := delivery.ComposedKey(*node, *module, *fp) - if err != nil { - return err - } - bucket, err := delivery.ComposedBucketFor(*node) - if err != nil { - return err - } - raw, err := os.ReadFile(*input) - if err != nil { - return err - } - var in compose.Input - if err := json.Unmarshal(raw, &in); err != nil { - return fmt.Errorf("the inputs in %s are not a compose.Input: %w", *input, err) - } - all, _, err := compose.Compositions(in, *node, map[string]string{*module: *fp}, compose.Reading) - if err != nil { - return err - } - c, ok := all[key2] - if !ok { - return fmt.Errorf("%s does not run %s: its composition holds none of it", *node, *module) - } - c, objects, err := c.ByDigest() - if err != nil { - return err - } - if c, err = delivery.SignComposed(c, key); err != nil { - return err - } - encoded, err := delivery.EncodeComposed(c) - if err != nil { - return err - } - digest, err := delivery.ComposedDigestOf(c) - if err != nil { - return err - } - out := printed{Bucket: bucket, Key: key2, Digest: digest, Composition: encoded, - Next: fmt.Sprintf("write want.%s.%s naming fp %s and composed %s with the operator's credential, and record each act in the hand-act log", *node, *module, *fp, digest)} - for d := range objects { - out.Objects = append(out.Objects, d) - } - if *write { - if err := writeIt(*url, *creds, *node, bucket, key2, encoded, digest, objects); err != nil { - return err + var req composer.Request + if strings.TrimSpace(string(raw)) == "" { + if *catalogueDir == "" { + fmt.Fprintln(stderr, "mesh-compose: no request on standard input and no -catalogue") + return 1 } - out.Written = true + req = composer.Request{V: composer.Version} + } else if req, err = composer.Decode(raw); err != nil { + fmt.Fprintln(stderr, "mesh-compose:", err) + return 1 + } + if *catalogueDir != "" { + read, err := readCatalogue(*catalogueDir) + if err != nil { + fmt.Fprintln(stderr, "mesh-compose:", err) + return 1 + } + if len(req.Cases) == 0 { + req.Cases = []composer.Case{{Name: "catalogue", Manifests: map[string]composer.Manifest{}}} + } + if req.Cases[0].Manifests == nil { + req.Cases[0].Manifests = map[string]composer.Manifest{} + } + for name, m := range read { + if _, given := req.Cases[0].Manifests[name]; !given { + req.Cases[0].Manifests[name] = m + } + } + // The catalogue's list of predefined keys is the one its manifests' fallbacks are judged against (ADR 0348 §7). + if req.Predefined == nil { + list, err := catalogue.ReadPredefined(*catalogueDir) + if err != nil { + fmt.Fprintln(stderr, "mesh-compose:", err) + return 1 + } + req.Predefined = list + } + } + req.Machines = append(req.Machines, machines...) + if *depth != "" { + req.Depth = *depth + } + if *declarations { + req.Declarations = true + } + v, err := composer.Run(req) + if err != nil { + fmt.Fprintln(stderr, "mesh-compose:", err) + return 1 } enc := json.NewEncoder(stdout) - enc.SetIndent("", " ") - return enc.Encode(out) + if err := enc.Encode(v); err != nil { + fmt.Fprintln(stderr, "mesh-compose: the verdict cannot be written:", err) + return 1 + } + return 0 } -func readKey(path string) (ed25519.PrivateKey, error) { - raw, err := os.ReadFile(path) +// readCatalogue reads every module.json of a catalogue checkout by its module's name: the root's and each of +// modules/*/. A manifest whose name cannot be read is kept under its directory's name, so the reader's refusal is said. +func readCatalogue(dir string) (map[string]composer.Manifest, error) { + paths, err := filepath.Glob(filepath.Join(dir, "modules", "*", "module.json")) if err != nil { return nil, err } - block, _ := pem.Decode(raw) - if block == nil { - return nil, fmt.Errorf("%s holds no PEM block", path) + if _, err := os.Stat(filepath.Join(dir, "module.json")); err == nil { + paths = append(paths, filepath.Join(dir, "module.json")) } - parsed, err := x509.ParsePKCS8PrivateKey(block.Bytes) - if err != nil { - return nil, fmt.Errorf("%s: %w", path, err) + if len(paths) == 0 { + return nil, fmt.Errorf("the catalogue %s holds no module.json", dir) } - key, ok := parsed.(ed25519.PrivateKey) - if !ok { - return nil, fmt.Errorf("%s is not an ed25519 key", path) - } - return key, nil -} - -// writeIt writes the files by digest, then the composition, then its digest: a reader that finds the digest finds the -// composition and its files already there. -func writeIt(url, creds, node, bucket, key string, encoded []byte, digest string, objects map[string][]byte) error { - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) - defer cancel() - nc, err := nats.Connect(url, nats.UserCredentials(creds), nats.Name("mesh-compose")) - if err != nil { - return err - } - defer nc.Close() - js, err := jetstream.New(nc) - if err != nil { - return err - } - if len(objects) > 0 { - filesBucket, err := declaration.FileBucketFor(node) + sort.Strings(paths) + out := map[string]composer.Manifest{} + for _, p := range paths { + raw, err := os.ReadFile(p) if err != nil { - return err + return nil, err } - store, err := js.ObjectStore(ctx, filesBucket) - if err != nil { - return fmt.Errorf("the node's object bucket %s: %w", filesBucket, err) + var probe struct { + Module string `json:"module"` } - for d, body := range objects { - name, err := declaration.ObjectName(d) - if err != nil { - return err - } - if _, err := store.PutBytes(ctx, name, body); err != nil { - return fmt.Errorf("the file %s: %w", d, err) - } + name := filepath.Base(filepath.Dir(p)) + if json.Unmarshal(raw, &probe) == nil && probe.Module != "" { + name = probe.Module } + if _, twice := out[name]; twice { + return nil, errors.New("the catalogue " + dir + " defines " + name + " twice") + } + out[name] = composer.Manifest{Raw: raw} } - composed, err := js.KeyValue(ctx, bucket) - if err != nil { - return fmt.Errorf("the bucket %s: %w", bucket, err) - } - if _, err := composed.Put(ctx, key, encoded); err != nil { - return fmt.Errorf("%s in %s: %w", key, bucket, err) - } - value, err := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digest, Judgement: delivery.JudgedCannot, - Words: "composed by hand with mesh-compose (ADR 0539 clause 10): the operator writes the want"}) - if err != nil { - return err - } - digests, err := js.KeyValue(ctx, delivery.ControllerComposedBucket) - if err != nil { - return fmt.Errorf("the bucket %s: %w", delivery.ControllerComposedBucket, err) - } - if _, err := digests.Put(ctx, key, value); err != nil { - return fmt.Errorf("%s in %s: %w", key, delivery.ControllerComposedBucket, err) - } - return nil + return out, nil } diff --git a/go/cmd/mesh-compose/main_test.go b/go/cmd/mesh-compose/main_test.go index f18c0f0..c68ff93 100644 --- a/go/cmd/mesh-compose/main_test.go +++ b/go/cmd/mesh-compose/main_test.go @@ -2,144 +2,346 @@ package main import ( "bytes" - "crypto/ed25519" - "crypto/rand" - "crypto/x509" - "encoding/base64" "encoding/json" - "encoding/pem" + "go/build" "os" + "os/exec" "path/filepath" + "reflect" + "runtime" "sort" "strings" "testing" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/compose" - "git.novox.be/novox/mesh-sdk/go/delivery" - "git.novox.be/novox/mesh-sdk/go/overlay" + "git.novox.be/novox/mesh-sdk/go/composer" + "git.novox.be/novox/mesh-sdk/go/internal/beside" ) -// aMesh is the Input of mesh-controller's aMesh, as mesh-sdk's compose tests make it (compose/fixture_test.go): two -// machines on the private network, the module the mesh provides for itself assigned to both. The manifest is the one -// the controller provides (overlay.Manifest), read back as the catalogue reads it. -func aMesh(t *testing.T) compose.Input { +// captured is every manifest of the catalogue captured in testdata/beside (internal/beside), raw, by module name. +func captured(t *testing.T) map[string]composer.Manifest { t.Helper() - var m catalogue.Manifest - b, _ := json.Marshal(overlay.Manifest()) - if err := json.Unmarshal(b, &m); err != nil { + read, err := readCatalogue(filepath.Dir(beside.Catalogue(t))) + if err != nil { t.Fatal(err) } - in := compose.Input{Catalogue: map[string]catalogue.Manifest{overlay.Name: m}} - for i, name := range []string{"anchor", "laptop"} { - pub, _, _ := ed25519.GenerateKey(rand.Reader) - in.Nodes = append(in.Nodes, compose.Node{Name: name, - Place: compose.Place{Key: base64.StdEncoding.EncodeToString(pub), Endpoint: name + ".example:51820", Site: "here", Hub: name == "anchor", Address: []string{"10.77.0.1", "10.77.0.2"}[i]}, - Assigned: []string{overlay.Name}, Capabilities: map[string]bool{"container-runtime": true, "wireguard": true, "systemd": true}}) + if len(read) < 100 { + t.Fatalf("the captured catalogue holds %d manifests, fewer than the catalogue's", len(read)) } - return in + return read } -func writeKey(t *testing.T, dir string) ed25519.PublicKey { +// meshOf is a mesh of three machines over the captured catalogue, each assigned real modules, the input the assigner +// composes with (nodes and their assignments, holdings from the claims): anchor runs the bus and the store, laptop a +// desktop's modules, ace a module whose requirement is provided elsewhere. +func meshOf() compose.Input { + return compose.Input{Nodes: []compose.Node{ + {Name: "anchor", Assigned: []string{"nats", "postgres"}}, + {Name: "laptop", Assigned: []string{"sudo", "zsh"}}, + {Name: "ace", Assigned: []string{"mesh-reads", "nats"}}, + }} +} + +func request(t *testing.T, depth string) composer.Request { t.Helper() - pub, priv, _ := ed25519.GenerateKey(rand.Reader) - der, err := x509.MarshalPKCS8PrivateKey(priv) - if err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(dir, "operator.key"), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der}), 0o600); err != nil { - t.Fatal(err) - } - return pub + return composer.Request{V: composer.Version, Input: meshOf(), Depth: depth, HoldingsFromClaims: true, + Machines: []string{"anchor", "laptop", "ace"}, + Cases: []composer.Case{{Name: "trunk", Manifests: captured(t)}}} } -// 1943/AC5: mesh-compose, given the kept inputs, a node, a module and its fingerprint, prints that module's -// composition for that node signed by the operator's key and its digest: its resources are the controller's for that -// module on that node (compose.Compose), and it verifies against the operator's key. Without --write it writes nothing. -func TestMeshComposeGivesTheControllersResourcesSignedByTheOperator(t *testing.T) { - dir := t.TempDir() - in := aMesh(t) - raw, err := json.Marshal(in) - if err != nil { - t.Fatal(err) - } - input := filepath.Join(dir, "input.json") - if err := os.WriteFile(input, raw, 0o600); err != nil { - t.Fatal(err) - } - operator := writeKey(t, dir) - fp := delivery.SourcePrefix2 + strings.Repeat("a", 64) - - var out bytes.Buffer - if err := run([]string{"-input", input, "-node", "laptop", "-module", overlay.Name, "-fp", fp, - "-key", filepath.Join(dir, "operator.key")}, &out); err != nil { - t.Fatal(err) - } - var printed struct { - Key string `json:"key"` - Bucket string `json:"bucket"` - Digest string `json:"digest"` - Composition json.RawMessage `json:"composition"` - Written bool `json:"written"` - } - if err := json.Unmarshal(out.Bytes(), &printed); err != nil { - t.Fatalf("%s: %v", out.String(), err) - } - c, err := delivery.DecodeComposed(printed.Composition) - if err != nil { - t.Fatal(err) - } - if digest, err := delivery.VerifyComposed(c, []ed25519.PublicKey{operator}); err != nil || digest != printed.Digest { - t.Errorf("the composition verifies %q, %v against the operator's key; printed %s", digest, err, printed.Digest) - } - wantKey, _ := delivery.ComposedKey("laptop", overlay.Name, fp) - if printed.Key != wantKey || printed.Bucket != "composed-laptop" || printed.Written { - t.Errorf("printed key %s in %s, written %v", printed.Key, printed.Bucket, printed.Written) - } - - d, _, err := compose.Compose(in, "laptop") - if err != nil { - t.Fatal(err) - } - var want, got []string - for _, r := range d.Resources { - want = append(want, r["id"].(string)) - } - for _, r := range c.Resources { - var head struct{ ID string } - _ = json.Unmarshal(r, &head) - got = append(got, head.ID) - } - sort.Strings(want) - sort.Strings(got) - if len(got) == 0 || strings.Join(got, " ") != strings.Join(want, " ") { - t.Errorf("mesh-compose gives %v, the controller's composition %v", got, want) +func runJSON(t *testing.T, args []string, in []byte) (int, composer.Verdict, string) { + t.Helper() + var out, errOut bytes.Buffer + code := run(args, bytes.NewReader(in), &out, &errOut) + var v composer.Verdict + if code == 0 { + if err := json.Unmarshal(out.Bytes(), &v); err != nil { + t.Fatalf("the verdict is not JSON: %v\n%s", err, out.String()) + } } + return code, v, errOut.String() } -// 1943/AC5: mesh-compose refuses what it cannot compose, naming why: no key, a fingerprint not of the new form, a -// module the node does not run, a node the mesh does not have. -func TestMeshComposeRefuses(t *testing.T) { +func encode(t *testing.T, r composer.Request) []byte { + t.Helper() + raw, err := json.Marshal(r) + if err != nil { + t.Fatal(err) + } + return raw +} + +// 1621/AC1: the command reads the plain input and the manifests on standard input, validates every manifest, composes +// the machines named and writes one verdict: per machine composes or the reason, per manifest accepted or the field +// refused. +func TestTheCommandWritesOneVerdictPerManifestAndMachine(t *testing.T) { + r := request(t, composer.DepthResolve) + r.Cases[0].Manifests["broken"] = composer.Manifest{Raw: json.RawMessage(`{"module":"broken","version":"1","shares":["x"]}`)} + code, v, stderr := runJSON(t, nil, encode(t, r)) + if code != 0 { + t.Fatalf("exit %d: %s", code, stderr) + } + if len(v.Cases) != 1 || v.Cases[0].Name != "trunk" || v.SDK == "" { + t.Fatalf("verdict = %+v", v) + } + c := v.Cases[0] + if got := c.Manifests["broken"]; got.Accepted || !strings.Contains(got.Refused, `"shares"`) { + t.Errorf("the manifest with an unknown field reads %+v, want refused naming the field", got) + } + if got := c.Manifests["nats"]; !got.Accepted { + t.Errorf("nats reads %+v, want accepted", got) + } + if len(c.Machines) != 3 { + t.Fatalf("machines = %v, want the three named", c.Machines) + } + for name, m := range c.Machines { + if !m.Composes && m.Reason == "" { + t.Errorf("%s does not compose and says no reason", name) + } + } +} + +// 1621/AC3: over the captured catalogue and a mesh of three machines, the command's verdict equals, machine for +// machine and manifest for manifest, what the linked reader, resolution and composition answer, at both depths. +func TestTheCommandAnswersWhatTheLinkedCompositionAnswers(t *testing.T) { + for _, depth := range []string{composer.DepthResolve, composer.DepthCompose} { + r := request(t, depth) + code, v, stderr := runJSON(t, nil, encode(t, r)) + if code != 0 { + t.Fatalf("%s: exit %d: %s", depth, code, stderr) + } + in := meshOf() + in.Catalogue = map[string]catalogue.Manifest{} + for name, m := range r.Cases[0].Manifests { + // A request without a list of predefined keys reads a fallback as the indexer does (Request.Predefined). + parsed, err := catalogue.ParseManifestForInputs(m.Raw) + got := v.Cases[0].Manifests[name] + if (err == nil) != got.Accepted || (err != nil && err.Error() != got.Refused) { + t.Errorf("%s: the manifest of %s: linked %v, command %+v", depth, name, err, got) + } + if err == nil { + in.Catalogue[name] = parsed + } + } + in.Holdings = composer.HoldingsOf(in.Catalogue, in.Nodes) + composed := 0 + for _, node := range r.Machines { + got := v.Cases[0].Machines[node] + var want composer.Machine + if depth == composer.DepthResolve { + if _, err := compose.Resolve(in, node); err != nil { + want = composer.Machine{Reason: err.Error()} + } else { + want = composer.Machine{Composes: true} + } + } else { + d, problems, err := compose.Compose(in, node) + switch { + case err != nil: + want = composer.Machine{Reason: err.Error()} + default: + want = composer.Machine{Composes: len(problems) == 0, Problems: problems, Refusals: d.Refusals()} + if len(problems) > 0 { + want.Reason = "the node-engine's validator refuses the declaration" + } + } + } + if !reflect.DeepEqual(got, want) { + t.Errorf("%s: %s: command %+v, linked %+v", depth, node, got, want) + } + if got.Composes { + composed++ + } else { + t.Logf("%s: %s does not compose: %s %v", depth, node, got.Reason, got.Problems) + } + } + t.Logf("%s: %d of %d machines compose", depth, composed, len(r.Machines)) + } +} + +// 1621 note 6: a person composes a catalogue checkout (repository@ref checked out or archived: the command fetches +// nothing) for a node and reads its declaration: each path with its kind, owner and mode. +func TestACatalogueCheckoutIsComposedForANode(t *testing.T) { dir := t.TempDir() - raw, _ := json.Marshal(aMesh(t)) - input := filepath.Join(dir, "input.json") - _ = os.WriteFile(input, raw, 0o600) - writeKey(t, dir) - key := filepath.Join(dir, "operator.key") - fp := delivery.SourcePrefix2 + strings.Repeat("a", 64) + if err := os.MkdirAll(filepath.Join(dir, "modules", "demo"), 0o755); err != nil { + t.Fatal(err) + } + manifest := `{"module": "demo", "version": "1", "resources": [{"id": "state", "type": "directory", ` + + `"path": "/var/lib/demo", "mode": "0750", "owner": "root"}]}` + if err := os.WriteFile(filepath.Join(dir, "modules", "demo", "module.json"), []byte(manifest), 0o644); err != nil { + t.Fatal(err) + } + in := encode(t, composer.Request{V: composer.Version, Input: compose.Input{Nodes: []compose.Node{{Name: "laptop", Assigned: []string{"demo"}}}}}) + code, v, stderr := runJSON(t, []string{"-catalogue", dir, "-node", "laptop", "-declarations"}, in) + if code != 0 { + t.Fatalf("exit %d: %s", code, stderr) + } + m := v.Cases[0].Machines["laptop"] + want := []composer.Path{{Path: "/var/lib/demo", Kind: "directory", Owner: "root", Mode: "0750", Module: ""}} + if !m.Composes || len(m.Declared) != 1 || m.Declared[0].Path != want[0].Path || m.Declared[0].Mode != "0750" || + m.Declared[0].Owner != "root" || m.Declared[0].Kind != "directory" { + t.Fatalf("laptop = %+v, want it composed declaring %+v", m, want) + } +} + +// A run step hands the command no standard input: the request is read from the file -request names, and a file that is +// not there is a request it cannot read. +func TestTheRequestIsReadFromAFile(t *testing.T) { + file := filepath.Join(t.TempDir(), "request.json") + r := composer.Request{V: composer.Version, Cases: []composer.Case{{Name: "only", Manifests: map[string]composer.Manifest{ + "demo": {Raw: json.RawMessage(`{"module":"demo","version":"1"}`)}}}}} + if err := os.WriteFile(file, encode(t, r), 0o600); err != nil { + t.Fatal(err) + } + code, v, stderr := runJSON(t, []string{"-request", file}, nil) + if code != 0 || len(v.Cases) != 1 || !v.Cases[0].Manifests["demo"].Accepted { + t.Fatalf("exit %d (%s), verdict %+v", code, stderr, v) + } + if code, _, stderr := runJSON(t, []string{"-request", file + ".gone"}, nil); code != 1 || !strings.Contains(stderr, "cannot be read") { + t.Errorf("a request file that is not there: exit %d, %q", code, stderr) + } +} + +// The exit codes: 1 for a request that cannot be read or run, 2 for wrong flags, each said on standard error. +func TestTheExitCodesSayWhatFailed(t *testing.T) { + unfilled := encode(t, composer.Request{V: composer.Version, Cases: []composer.Case{{Name: "c", Manifests: map[string]composer.Manifest{"x": {Digest: "abc"}}}}}) for _, c := range []struct { + name string args []string - why string + in string + code int + says string }{ - {[]string{"-input", input, "-node", "laptop", "-module", overlay.Name, "-fp", fp}, "-key"}, - {[]string{"-input", input, "-node", "laptop", "-module", overlay.Name, "-fp", "src2:1", "-key", key}, "fingerprint"}, - {[]string{"-input", input, "-node", "laptop", "-module", "mesh-mail", "-fp", fp, "-key", key}, "does not run mesh-mail"}, - {[]string{"-input", input, "-node", "nowhere", "-module", overlay.Name, "-fp", fp, "-key", key}, "nowhere"}, - {[]string{"-input", input, "-node", "laptop", "-module", overlay.Name, "-fp", fp, "-key", key, "-write"}, "-creds"}, + {"not JSON", nil, "{", 1, "not JSON"}, + {"another version", nil, `{"v":2}`, 1, "version 2"}, + {"a digest left unfilled", nil, string(unfilled), 1, "digest"}, + {"an unknown depth", nil, `{"v":1,"depth":"deep"}`, 1, "depth"}, + {"nothing to read", nil, "", 1, "no request"}, + {"a wrong flag", []string{"-nope"}, "", 2, "nope"}, + {"an argument", []string{"extra"}, "", 2, "unexpected"}, } { - var out bytes.Buffer - if err := run(c.args, &out); err == nil || !strings.Contains(err.Error(), c.why) { - t.Errorf("mesh-compose %v = %v, want a refusal saying %q", c.args, err, c.why) + var out, errOut bytes.Buffer + if got := run(c.args, strings.NewReader(c.in), &out, &errOut); got != c.code || !strings.Contains(errOut.String(), c.says) { + t.Errorf("%s: exit %d saying %q, want %d saying %q", c.name, got, errOut.String(), c.code, c.says) + } + } +} + +// sdkPackages is every package of this module the command imports, directly or not, with the packages each imports. +func sdkPackages(t *testing.T) map[string][]string { + t.Helper() + _, file, _, _ := runtime.Caller(0) + root := filepath.Join(filepath.Dir(file), "..", "..") + const module = "git.novox.be/novox/mesh-sdk/go" + out := map[string][]string{} + var walk func(path string) + walk = func(path string) { + if _, done := out[path]; done { + return + } + rel := strings.TrimPrefix(strings.TrimPrefix(path, module), "/") + pkg, err := build.Default.ImportDir(filepath.Join(root, rel), 0) + if err != nil { + t.Fatalf("%s: %v", path, err) + } + out[path] = pkg.Imports + for _, imp := range pkg.Imports { + if strings.HasPrefix(imp, module+"/") { + walk(imp) + } + } + } + walk(composer.Package) + return out +} + +// 1621/AC2 and AC4: the command imports, through every package of mesh-sdk it reaches, only the standard library and +// mesh-sdk's own packages (so `go build` from mesh-sdk's module needs nothing beyond its lock), and none that reaches +// a bus, a store, a network service or another process. +func TestTheCommandImportsNoBusStoreNetworkOrProcess(t *testing.T) { + forbidden := []string{"net/http", "net/rpc", "net/smtp", "os/exec", "database/sql", "plugin", "syscall"} + for path, imports := range sdkPackages(t) { + for _, imp := range imports { + std := !strings.Contains(strings.SplitN(imp, "/", 2)[0], ".") + if !std && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-sdk/go/") { + t.Errorf("%s imports %s, which is neither the standard library nor mesh-sdk", path, imp) + } + for _, f := range forbidden { + if imp == f { + t.Errorf("%s imports %s", path, imp) + } + } + } + } +} + +// 1621/AC2: run with an empty network namespace, the command still writes its verdict: it needs no network. The test +// runs its own binary again under `unshare --net` as an unprivileged user's namespace; where the kernel or the sandbox +// allows no new namespace (a build module's run step is already one with no network), it says so and skips. +func TestTheCommandRunsWithNoNetwork(t *testing.T) { + if os.Getenv("MESH_COMPOSE_HELPER") == "1" { + os.Exit(run(nil, os.Stdin, os.Stdout, os.Stderr)) + } + unshare, err := exec.LookPath("unshare") + if err != nil { + t.Skip("no unshare on this machine, so no empty network namespace can be made here") + } + if out, err := exec.Command(unshare, "--net", "--map-root-user", "true").CombinedOutput(); err != nil { + t.Skipf("this machine makes no unprivileged network namespace (%v: %s); a run step's own has no network", err, out) + } + r := request(t, composer.DepthResolve) + cmd := exec.Command(unshare, "--net", "--map-root-user", os.Args[0], "-test.run=^TestTheCommandRunsWithNoNetwork$") + cmd.Env = []string{"MESH_COMPOSE_HELPER=1"} + cmd.Stdin = bytes.NewReader(encode(t, r)) + var out, errOut bytes.Buffer + cmd.Stdout, cmd.Stderr = &out, &errOut + if err := cmd.Run(); err != nil { + t.Fatalf("the command in an empty network namespace: %v: %s", err, errOut.String()) + } + var v composer.Verdict + if err := json.Unmarshal(out.Bytes(), &v); err != nil || len(v.Cases) != 1 || len(v.Cases[0].Machines) != 3 { + t.Fatalf("no verdict from the command with no network: %v\n%s", err, out.String()) + } + names := make([]string, 0, len(v.Cases[0].Machines)) + for n := range v.Cases[0].Machines { + names = append(names, n) + } + sort.Strings(names) + t.Logf("composed with no network: %v", names) +} + +// -catalogue judges a fallback to a predefined key against the predefined.json at the checkout's root (ADR 0348 §7): +// xterm (mesh-catalog at 835bf22b) falls back to "font-size", accepted where the root lists it and refused where the +// root's list does not. +func TestACatalogueCheckoutJudgesFallbacksAgainstItsPredefinedKeys(t *testing.T) { + raw, err := os.ReadFile("../../catalogue/testdata/catalogue-835bf22b-xterm.module.json") + if err != nil { + t.Fatal(err) + } + for _, c := range []struct { + list string + want bool + }{ + {`{"font-size": {"type": "number", "description": "the desktop's font size"}}`, true}, + {`{"font-family": {"type": "string", "description": "the desktop's font"}}`, false}, + } { + dir := t.TempDir() + if err := os.MkdirAll(filepath.Join(dir, "modules", "xterm"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "modules", "xterm", "module.json"), raw, 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, catalogue.PredefinedFile), []byte(c.list), 0o600); err != nil { + t.Fatal(err) + } + code, v, stderr := runJSON(t, []string{"-catalogue", dir}, nil) + if code != 0 { + t.Fatalf("exit %d: %s", code, stderr) + } + if got := v.Cases[0].Manifests["xterm"]; got.Accepted != c.want { + t.Errorf("predefined.json %s: xterm %+v, want accepted %v", c.list, got, c.want) } } } diff --git a/go/cmd/mesh-compose/reads-outside b/go/cmd/mesh-compose/reads-outside new file mode 100644 index 0000000..f6b2d87 --- /dev/null +++ b/go/cmd/mesh-compose/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/go/cmd/mesh-hand-compose/main.go b/go/cmd/mesh-hand-compose/main.go new file mode 100644 index 0000000..cf59e98 --- /dev/null +++ b/go/cmd/mesh-hand-compose/main.go @@ -0,0 +1,209 @@ +// mesh-hand-compose composes one module for one node by hand, from the inputs the controller keeps, and signs it with the +// operator's key, for the hand path of novox/hq ADR 0539 clause 10: a fingerprint the controller never composed (a +// hand build), while the controller cannot compose its own fix. It composes with mesh-sdk's composition, the one the +// controller writes with (compose.Compositions), and prints the composition, its key and its digest; only with -write +// does it write them, with the operator's credential: the composition into the node's bucket composed-, each +// file over 64 KiB into the node's object bucket, and the digest into mesh-controller_composed. The want is the +// operator's next act, naming that digest; each act is recorded in the hand-act log. +// +// mesh-hand-compose -input -node -module -fp -key +// [-write -nats -creds ] +// +// The inputs are a compose.Input as JSON. The key is the operator's ed25519 private key, PKCS#8 in PEM; the +// node-engine trusts it beside the controller's (ADR 0539 clause 3). +package main + +import ( + "context" + "crypto/ed25519" + "crypto/x509" + "encoding/json" + "encoding/pem" + "errors" + "flag" + "fmt" + "io" + "os" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/compose" + "git.novox.be/novox/mesh-sdk/go/declaration" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +func main() { + if err := run(os.Args[1:], os.Stdout); err != nil { + fmt.Fprintln(os.Stderr, "mesh-hand-compose:", err) + os.Exit(1) + } +} + +// printed is what mesh-hand-compose prints: where the composition goes, its digest, the composition, and whether it was +// written. +type printed struct { + Bucket string `json:"bucket"` + Key string `json:"key"` + Digest string `json:"digest"` + Composition json.RawMessage `json:"composition"` + Objects []string `json:"objects,omitempty"` + Written bool `json:"written"` + Next string `json:"next"` +} + +func run(args []string, stdout io.Writer) error { + fs := flag.NewFlagSet("mesh-hand-compose", flag.ContinueOnError) + input := fs.String("input", "", "the controller's kept inputs, a compose.Input as JSON") + node := fs.String("node", "", "the node to compose for") + module := fs.String("module", "", "the module to compose") + fp := fs.String("fp", "", "the module's fingerprint, src2: and 64 hex digits") + keyFile := fs.String("key", "", "the operator's ed25519 private key, PKCS#8 in PEM") + write := fs.Bool("write", false, "write the composition, its files and its digest, with the operator's credential") + url := fs.String("nats", nats.DefaultURL, "with -write: the bus") + creds := fs.String("creds", "", "with -write: the operator's credential file") + if err := fs.Parse(args); err != nil { + return err + } + switch { + case *input == "" || *node == "" || *module == "" || *fp == "": + return errors.New("-input, -node, -module and -fp are each required") + case *keyFile == "": + return errors.New("-key is required: a composition made by hand is signed by the operator's key, and the node-engine applies no other") + case *write && *creds == "": + return errors.New("-write needs -creds: the operator's credential is the only hand writer of a composition") + } + key, err := readKey(*keyFile) + if err != nil { + return err + } + key2, err := delivery.ComposedKey(*node, *module, *fp) + if err != nil { + return err + } + bucket, err := delivery.ComposedBucketFor(*node) + if err != nil { + return err + } + raw, err := os.ReadFile(*input) + if err != nil { + return err + } + var in compose.Input + if err := json.Unmarshal(raw, &in); err != nil { + return fmt.Errorf("the inputs in %s are not a compose.Input: %w", *input, err) + } + all, _, err := compose.Compositions(in, *node, map[string]string{*module: *fp}, compose.Reading) + if err != nil { + return err + } + c, ok := all[key2] + if !ok { + return fmt.Errorf("%s does not run %s: its composition holds none of it", *node, *module) + } + c, objects, err := c.ByDigest() + if err != nil { + return err + } + if c, err = delivery.SignComposed(c, key); err != nil { + return err + } + encoded, err := delivery.EncodeComposed(c) + if err != nil { + return err + } + digest, err := delivery.ComposedDigestOf(c) + if err != nil { + return err + } + out := printed{Bucket: bucket, Key: key2, Digest: digest, Composition: encoded, + Next: fmt.Sprintf("write want.%s.%s naming fp %s and composed %s with the operator's credential, and record each act in the hand-act log", *node, *module, *fp, digest)} + for d := range objects { + out.Objects = append(out.Objects, d) + } + if *write { + if err := writeIt(*url, *creds, *node, bucket, key2, encoded, digest, objects); err != nil { + return err + } + out.Written = true + } + enc := json.NewEncoder(stdout) + enc.SetIndent("", " ") + return enc.Encode(out) +} + +func readKey(path string) (ed25519.PrivateKey, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + block, _ := pem.Decode(raw) + if block == nil { + return nil, fmt.Errorf("%s holds no PEM block", path) + } + parsed, err := x509.ParsePKCS8PrivateKey(block.Bytes) + if err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + key, ok := parsed.(ed25519.PrivateKey) + if !ok { + return nil, fmt.Errorf("%s is not an ed25519 key", path) + } + return key, nil +} + +// writeIt writes the files by digest, then the composition, then its digest: a reader that finds the digest finds the +// composition and its files already there. +func writeIt(url, creds, node, bucket, key string, encoded []byte, digest string, objects map[string][]byte) error { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + nc, err := nats.Connect(url, nats.UserCredentials(creds), nats.Name("mesh-hand-compose")) + if err != nil { + return err + } + defer nc.Close() + js, err := jetstream.New(nc) + if err != nil { + return err + } + if len(objects) > 0 { + filesBucket, err := declaration.FileBucketFor(node) + if err != nil { + return err + } + store, err := js.ObjectStore(ctx, filesBucket) + if err != nil { + return fmt.Errorf("the node's object bucket %s: %w", filesBucket, err) + } + for d, body := range objects { + name, err := declaration.ObjectName(d) + if err != nil { + return err + } + if _, err := store.PutBytes(ctx, name, body); err != nil { + return fmt.Errorf("the file %s: %w", d, err) + } + } + } + composed, err := js.KeyValue(ctx, bucket) + if err != nil { + return fmt.Errorf("the bucket %s: %w", bucket, err) + } + if _, err := composed.Put(ctx, key, encoded); err != nil { + return fmt.Errorf("%s in %s: %w", key, bucket, err) + } + value, err := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digest, Judgement: delivery.JudgedCannot, + Words: "composed by hand with mesh-hand-compose (ADR 0539 clause 10): the operator writes the want"}) + if err != nil { + return err + } + digests, err := js.KeyValue(ctx, delivery.ControllerComposedBucket) + if err != nil { + return fmt.Errorf("the bucket %s: %w", delivery.ControllerComposedBucket, err) + } + if _, err := digests.Put(ctx, key, value); err != nil { + return fmt.Errorf("%s in %s: %w", key, delivery.ControllerComposedBucket, err) + } + return nil +} diff --git a/go/cmd/mesh-hand-compose/main_test.go b/go/cmd/mesh-hand-compose/main_test.go new file mode 100644 index 0000000..96ffd48 --- /dev/null +++ b/go/cmd/mesh-hand-compose/main_test.go @@ -0,0 +1,145 @@ +package main + +import ( + "bytes" + "crypto/ed25519" + "crypto/rand" + "crypto/x509" + "encoding/base64" + "encoding/json" + "encoding/pem" + "os" + "path/filepath" + "sort" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/compose" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/overlay" +) + +// aMesh is the Input of mesh-controller's aMesh, as mesh-sdk's compose tests make it (compose/fixture_test.go): two +// machines on the private network, the module the mesh provides for itself assigned to both. The manifest is the one +// the controller provides (overlay.Manifest), read back as the catalogue reads it. +func aMesh(t *testing.T) compose.Input { + t.Helper() + var m catalogue.Manifest + b, _ := json.Marshal(overlay.Manifest()) + if err := json.Unmarshal(b, &m); err != nil { + t.Fatal(err) + } + in := compose.Input{Catalogue: map[string]catalogue.Manifest{overlay.Name: m}} + for i, name := range []string{"anchor", "laptop"} { + pub, _, _ := ed25519.GenerateKey(rand.Reader) + in.Nodes = append(in.Nodes, compose.Node{Name: name, + Place: compose.Place{Key: base64.StdEncoding.EncodeToString(pub), Endpoint: name + ".example:51820", Site: "here", Hub: name == "anchor", Address: []string{"10.77.0.1", "10.77.0.2"}[i]}, + Assigned: []string{overlay.Name}, Capabilities: map[string]bool{"container-runtime": true, "wireguard": true, "systemd": true}}) + } + return in +} + +func writeKey(t *testing.T, dir string) ed25519.PublicKey { + t.Helper() + pub, priv, _ := ed25519.GenerateKey(rand.Reader) + der, err := x509.MarshalPKCS8PrivateKey(priv) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "operator.key"), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der}), 0o600); err != nil { + t.Fatal(err) + } + return pub +} + +// 1943/AC5: mesh-hand-compose, given the kept inputs, a node, a module and its fingerprint, prints that module's +// composition for that node signed by the operator's key and its digest: its resources are the controller's for that +// module on that node (compose.Compose), and it verifies against the operator's key. Without --write it writes nothing. +func TestMeshComposeGivesTheControllersResourcesSignedByTheOperator(t *testing.T) { + dir := t.TempDir() + in := aMesh(t) + raw, err := json.Marshal(in) + if err != nil { + t.Fatal(err) + } + input := filepath.Join(dir, "input.json") + if err := os.WriteFile(input, raw, 0o600); err != nil { + t.Fatal(err) + } + operator := writeKey(t, dir) + fp := delivery.SourcePrefix2 + strings.Repeat("a", 64) + + var out bytes.Buffer + if err := run([]string{"-input", input, "-node", "laptop", "-module", overlay.Name, "-fp", fp, + "-key", filepath.Join(dir, "operator.key")}, &out); err != nil { + t.Fatal(err) + } + var printed struct { + Key string `json:"key"` + Bucket string `json:"bucket"` + Digest string `json:"digest"` + Composition json.RawMessage `json:"composition"` + Written bool `json:"written"` + } + if err := json.Unmarshal(out.Bytes(), &printed); err != nil { + t.Fatalf("%s: %v", out.String(), err) + } + c, err := delivery.DecodeComposed(printed.Composition) + if err != nil { + t.Fatal(err) + } + if digest, err := delivery.VerifyComposed(c, []ed25519.PublicKey{operator}); err != nil || digest != printed.Digest { + t.Errorf("the composition verifies %q, %v against the operator's key; printed %s", digest, err, printed.Digest) + } + wantKey, _ := delivery.ComposedKey("laptop", overlay.Name, fp) + if printed.Key != wantKey || printed.Bucket != "composed-laptop" || printed.Written { + t.Errorf("printed key %s in %s, written %v", printed.Key, printed.Bucket, printed.Written) + } + + d, _, err := compose.Compose(in, "laptop") + if err != nil { + t.Fatal(err) + } + var want, got []string + for _, r := range d.Resources { + want = append(want, r["id"].(string)) + } + for _, r := range c.Resources { + var head struct{ ID string } + _ = json.Unmarshal(r, &head) + got = append(got, head.ID) + } + sort.Strings(want) + sort.Strings(got) + if len(got) == 0 || strings.Join(got, " ") != strings.Join(want, " ") { + t.Errorf("mesh-hand-compose gives %v, the controller's composition %v", got, want) + } +} + +// 1943/AC5: mesh-hand-compose refuses what it cannot compose, naming why: no key, a fingerprint not of the new form, a +// module the node does not run, a node the mesh does not have. +func TestMeshComposeRefuses(t *testing.T) { + dir := t.TempDir() + raw, _ := json.Marshal(aMesh(t)) + input := filepath.Join(dir, "input.json") + _ = os.WriteFile(input, raw, 0o600) + writeKey(t, dir) + key := filepath.Join(dir, "operator.key") + fp := delivery.SourcePrefix2 + strings.Repeat("a", 64) + for _, c := range []struct { + args []string + why string + }{ + {[]string{"-input", input, "-node", "laptop", "-module", overlay.Name, "-fp", fp}, "-key"}, + {[]string{"-input", input, "-node", "laptop", "-module", overlay.Name, "-fp", "src2:1", "-key", key}, "fingerprint"}, + {[]string{"-input", input, "-node", "laptop", "-module", "mesh-mail", "-fp", fp, "-key", key}, "does not run mesh-mail"}, + {[]string{"-input", input, "-node", "nowhere", "-module", overlay.Name, "-fp", fp, "-key", key}, "nowhere"}, + {[]string{"-input", input, "-node", "laptop", "-module", overlay.Name, "-fp", fp, "-key", key, "-write"}, "-creds"}, + } { + var out bytes.Buffer + if err := run(c.args, &out); err == nil || !strings.Contains(err.Error(), c.why) { + t.Errorf("mesh-hand-compose %v = %v, want a refusal saying %q", c.args, err, c.why) + } + } +} diff --git a/go/compose/composed.go b/go/compose/composed.go index 953b896..9fe66a3 100644 --- a/go/compose/composed.go +++ b/go/compose/composed.go @@ -13,7 +13,7 @@ import ( // the controller composes it, split into each module's composition at the fingerprint fps gives it and the node's own // (`.node`), by key (delivery.ComposedKey). Each resource says its module, whatever the node reported it reads. // A module fps gives no fingerprint for is not composed and is named in skipped, so a caller composing one module -// (mesh-compose) gets that one. Each module's Data says whether its manifest declares data. The compositions are not +// (mesh-hand-compose) gets that one. Each module's Data says whether its manifest declares data. The compositions are not // signed and not yet by digest: the writer does both (delivery.SignComposed, Composed.ByDigest). func Compositions(in Input, node string, fps map[string]string, mode Mode) (out map[string]delivery.Composed, skipped []string, err error) { reading := in diff --git a/go/composer/composer.go b/go/composer/composer.go new file mode 100644 index 0000000..a456a1a --- /dev/null +++ b/go/composer/composer.go @@ -0,0 +1,347 @@ +// Package composer is mesh-sdk's composition and validator as one function over a plain request (novox/hq ADR 0482 +// clause 4, issue 1621): what the command mesh-compose runs, so a program that links any one mesh-sdk can have the +// composition of another commit judged by building that commit's command and reading its verdict. +// +// **It reads its request and nothing else.** No bus, store, network, file or environment: the manifests come inside the +// request (raw, as their repository holds them), the rest of the mesh is a compose.Input, and the answer is a value. +// A manifest named only by its digest in the delivery indexer's bucket is filled before the run by Fill, by whoever +// holds the bus (the Go build module's compose job); Run refuses a digest left unfilled. +package composer + +import ( + "encoding/json" + "errors" + "fmt" + "slices" + "sort" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/compose" + "git.novox.be/novox/mesh-sdk/go/sdkversion" +) + +// Version is the version of a Request and of a Verdict; a reader refuses another. +const Version = 1 + +// Package is the import path of the command, which a compose job builds at a commit (in mesh-sdk's own module, or in +// a module that requires mesh-sdk, at the version it pins). +const Package = "git.novox.be/novox/mesh-sdk/go/cmd/mesh-compose" + +// The depths of a composition. +const ( + // DepthResolve resolves each machine's set over the rest of the mesh (compose.Resolve): what the assigner judges + // while the facts snapshot is not on the bus (issue 1461). + DepthResolve = "resolve" + // DepthCompose composes each machine's whole declaration and runs the node-engine's validator over it + // (compose.Compose). + DepthCompose = "compose" +) + +// Request is everything one run composes: the rest of the mesh, one or more cases of manifests composed over it, and +// the machines composed in each case. +type Request struct { + V int `json:"v"` + // Input is the composition's input without its catalogue (and without each node's kept manifests): the cases give + // those. A Catalogue or Kept it holds is ignored. + Input compose.Input `json:"input"` + // Cases are composed one after the other over Input, each with its own manifests; their names are unique. + Cases []Case `json:"cases"` + // Machines are the nodes composed in every case; none means the manifests are only read. + Machines []string `json:"machines,omitempty"` + // Depth is DepthResolve or DepthCompose; empty is DepthCompose. + Depth string `json:"depth,omitempty"` + // HoldingsFromClaims adds to Input's holdings every seat an assigned module claims, held on its node (the + // assigner's input, which has no holdings of record). + HoldingsFromClaims bool `json:"holdings_from_claims,omitempty"` + // Declarations asks each composed machine's declared paths (DepthCompose only). + Declarations bool `json:"declarations,omitempty"` + // Predefined is the list of predefined keys at the catalogue's root (predefined.json, novox/hq ADR 0348 §7) that + // every config value falling back to a key is judged against, when the asker has it (mesh-compose -catalogue + // reads it). Without it a fallback is read as catalogue.ParseManifestForInputs reads it: against the list the mesh + // published when this process holds one, else each key as its value declares it, for the merge gate judges the + // fallback in the catalogue and a manifest that only carries one is not refused here. + Predefined catalogue.PredefinedList `json:"predefined,omitempty"` +} + +// Case is one catalogue composed: every module's manifest by name, and per node the manifests kept at the build it runs. +type Case struct { + Name string `json:"name"` + Manifests map[string]Manifest `json:"manifests"` + Kept map[string]map[string]Manifest `json:"kept,omitempty"` +} + +// Manifest is a module's manifest as its repository holds it (Raw, the bytes), or the digest the delivery indexer keeps +// it under (Digest), which Fill replaces by the bytes. On the wire it is the manifest itself, or {"digest": "..."}. +type Manifest struct { + Raw json.RawMessage + Digest string +} + +// MarshalJSON writes the raw manifest, or {"digest": ...} when only the digest is known. +func (m Manifest) MarshalJSON() ([]byte, error) { + if len(m.Raw) > 0 { + return m.Raw, nil + } + return json.Marshal(map[string]string{"digest": m.Digest}) +} + +// UnmarshalJSON reads {"digest": "..."} (an object with that one key) as a digest, and any other value as the raw +// manifest. +func (m *Manifest) UnmarshalJSON(raw []byte) error { + var probe map[string]json.RawMessage + if json.Unmarshal(raw, &probe) == nil && len(probe) == 1 { + if d, ok := probe["digest"]; ok { + var digest string + if err := json.Unmarshal(d, &digest); err != nil { + return fmt.Errorf("a manifest's digest is not a string: %w", err) + } + *m = Manifest{Digest: digest} + return nil + } + } + *m = Manifest{Raw: slices.Clone(raw)} + return nil +} + +// Verdict is what one run answers: the mesh-sdk the command was built with, and each case's verdict in the order asked. +type Verdict struct { + V int `json:"v"` + SDK string `json:"sdk"` + Cases []CaseVerdict `json:"cases"` +} + +// CaseVerdict is one case's: every manifest read or refused, every machine composed or not. +type CaseVerdict struct { + Name string `json:"name"` + Manifests map[string]ManifestRead `json:"manifests"` + Machines map[string]Machine `json:"machines,omitempty"` +} + +// ManifestRead is one manifest as this mesh-sdk reads it: accepted, or refused with the reader's words. +type ManifestRead struct { + Accepted bool `json:"accepted"` + Refused string `json:"refused,omitempty"` +} + +// Machine is one machine composed: whether it composes (resolves, at DepthResolve), why not, the node-engine +// validator's problems and the composition's refusals (DepthCompose), and its declared paths when asked. +type Machine struct { + Composes bool `json:"composes"` + Reason string `json:"reason,omitempty"` + Problems []string `json:"problems,omitempty"` + Refusals []string `json:"refusals,omitempty"` + Declared []Path `json:"declared,omitempty"` +} + +// Path is one path a declaration puts on the machine, with its kind (the resource's type), owner, group and mode as +// declared, and the module it is of. +type Path struct { + Path string `json:"path"` + Kind string `json:"kind"` + Owner string `json:"owner,omitempty"` + Group string `json:"group,omitempty"` + Mode string `json:"mode,omitempty"` + Module string `json:"module,omitempty"` +} + +// ErrUnfilled is a manifest named by a digest that nobody replaced by its bytes. +var ErrUnfilled = errors.New("composer: a manifest is named by its digest and was not filled") + +// Decode reads a request; it refuses one that is not JSON or of another version. +func Decode(data []byte) (Request, error) { + var r Request + if err := json.Unmarshal(data, &r); err != nil { + return Request{}, fmt.Errorf("composer: the request is not JSON of its shape: %w", err) + } + if r.V != Version { + return Request{}, fmt.Errorf("composer: a request of version %d, this composer reads %d", r.V, Version) + } + return r, nil +} + +// Fill replaces every manifest named by its digest by the bytes get answers for that digest; a digest get cannot answer +// stops it, named. +func Fill(r *Request, get func(digest string) ([]byte, error)) error { + fill := func(m map[string]Manifest) error { + for name, man := range m { + if len(man.Raw) > 0 { + continue + } + raw, err := get(man.Digest) + if err != nil { + return fmt.Errorf("composer: the manifest of %s (digest %s): %w", name, man.Digest, err) + } + m[name] = Manifest{Raw: raw} + } + return nil + } + for _, c := range r.Cases { + if err := fill(c.Manifests); err != nil { + return err + } + for _, kept := range c.Kept { + if err := fill(kept); err != nil { + return err + } + } + } + return nil +} + +// Run composes every case of r and answers the verdict, its SDK this binary's mesh-sdk. An error is a request it +// cannot run (an unknown depth, a case named twice, a manifest left as a digest); a machine that does not compose is a +// verdict. +func Run(r Request) (Verdict, error) { + depth := r.Depth + if depth == "" { + depth = DepthCompose + } + if depth != DepthResolve && depth != DepthCompose { + return Verdict{}, fmt.Errorf("composer: the depth %q is not resolve or compose", r.Depth) + } + v := Verdict{V: Version, SDK: sdkversion.Version()} + seen := map[string]bool{} + for _, c := range r.Cases { + if seen[c.Name] { + return Verdict{}, fmt.Errorf("composer: the case %q is named twice", c.Name) + } + seen[c.Name] = true + cv, err := runCase(r, c, depth) + if err != nil { + return Verdict{}, err + } + v.Cases = append(v.Cases, cv) + } + return v, nil +} + +// readManifest reads one manifest as this run reads every one: against the request's list of predefined keys when it +// has one, else as the indexer and the builder read it (Request.Predefined). +func readManifest(raw []byte, predefined catalogue.PredefinedList) (catalogue.Manifest, error) { + if predefined != nil { + return catalogue.ParseManifestWith(raw, predefined) + } + return catalogue.ParseManifestForInputs(raw) +} + +func parse(m map[string]Manifest, reads map[string]ManifestRead, predefined catalogue.PredefinedList) (map[string]catalogue.Manifest, error) { + out := map[string]catalogue.Manifest{} + for name, man := range m { + if len(man.Raw) == 0 { + return nil, fmt.Errorf("%w: %s (digest %s)", ErrUnfilled, name, man.Digest) + } + parsed, err := readManifest(man.Raw, predefined) + if reads != nil { + if err != nil { + reads[name] = ManifestRead{Refused: err.Error()} + } else { + reads[name] = ManifestRead{Accepted: true} + } + } + if err == nil { + out[name] = parsed + } + } + return out, nil +} + +func runCase(r Request, c Case, depth string) (CaseVerdict, error) { + cv := CaseVerdict{Name: c.Name, Manifests: map[string]ManifestRead{}} + shelf, err := parse(c.Manifests, cv.Manifests, r.Predefined) + if err != nil { + return CaseVerdict{}, err + } + in := r.Input + in.Catalogue = shelf + in.Nodes = slices.Clone(in.Nodes) + in.Holdings = slices.Clone(in.Holdings) + for i := range in.Nodes { + in.Nodes[i].Kept = nil + if kept := c.Kept[in.Nodes[i].Name]; len(kept) > 0 { + k, err := parse(kept, nil, r.Predefined) + if err != nil { + return CaseVerdict{}, err + } + in.Nodes[i].Kept = k + } + } + if r.HoldingsFromClaims { + in.Holdings = append(in.Holdings, HoldingsOf(shelf, in.Nodes)...) + } + if len(r.Machines) == 0 { + return cv, nil + } + cv.Machines = map[string]Machine{} + for _, node := range r.Machines { + cv.Machines[node] = machine(in, node, depth, r.Declarations) + } + return cv, nil +} + +// HoldingsOf is every seat an assigned module claims, held on the node it is assigned to, in node order: the holdings +// of an input that has none of record (the assigner's, mesh-delivery-assigner internal/assign input). +func HoldingsOf(shelf map[string]catalogue.Manifest, nodes []compose.Node) []catalogue.Held { + var out []catalogue.Held + sorted := slices.Clone(nodes) + sort.Slice(sorted, func(i, j int) bool { return sorted[i].Name < sorted[j].Name }) + for _, n := range sorted { + for _, mod := range n.Assigned { + for _, cl := range shelf[mod].Claims { + scope := cl.Scope + if seat, ok := catalogue.SeatNamed(cl.Name); ok && scope == "" { + scope = seat.Scope + } + out = append(out, catalogue.Held{Claim: cl.Name, Scope: scope, Node: n.Name, Module: mod}) + } + } + } + return out +} + +func machine(in compose.Input, node, depth string, declarations bool) Machine { + if depth == DepthResolve { + if _, err := compose.Resolve(in, node); err != nil { + return Machine{Reason: err.Error()} + } + return Machine{Composes: true} + } + d, problems, err := compose.Compose(in, node) + if err != nil { + return Machine{Reason: err.Error()} + } + m := Machine{Composes: len(problems) == 0, Problems: problems, Refusals: d.Refusals()} + if len(problems) > 0 { + m.Reason = "the node-engine's validator refuses the declaration" + } + if declarations { + m.Declared = pathsOf(d) + } + return m +} + +// pathsOf is every resource of a declaration that names a path, with its owner, group and mode as declared, sorted by +// path. +func pathsOf(d compose.Declared) []Path { + var out []Path + str := func(r map[string]any, key string) string { + if v, ok := r[key]; ok && v != nil { + return fmt.Sprint(v) + } + return "" + } + for _, r := range d.Resources { + p := str(r, "path") + if p == "" { + continue + } + out = append(out, Path{Path: p, Kind: str(r, "type"), Owner: str(r, "owner"), Group: str(r, "group"), + Mode: str(r, "mode"), Module: d.ModuleOf[str(r, "id")]}) + } + sort.Slice(out, func(i, j int) bool { + if out[i].Path != out[j].Path { + return out[i].Path < out[j].Path + } + return out[i].Kind < out[j].Kind + }) + return out +} diff --git a/go/composer/composer_test.go b/go/composer/composer_test.go new file mode 100644 index 0000000..5e26900 --- /dev/null +++ b/go/composer/composer_test.go @@ -0,0 +1,137 @@ +package composer + +import ( + "encoding/json" + "errors" + "os" + "reflect" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/compose" +) + +const demo = `{"module": "demo", "version": "1", "claims": [{"name": "node-files"}]}` + +// A manifest named by its digest goes on the wire as {"digest": ...}, and Fill puts the bytes the indexer keeps in its +// place; a raw manifest goes as itself. +func TestAManifestIsItsBytesOrItsDigest(t *testing.T) { + r := Request{V: Version, Cases: []Case{{Name: "change", Manifests: map[string]Manifest{ + "demo": {Digest: "d1"}, "raw": {Raw: json.RawMessage(`{"module":"raw","version":"1"}`)}}, + Kept: map[string]map[string]Manifest{"laptop": {"demo": {Digest: "d1"}}}}}} + wire, err := json.Marshal(r) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(wire), `"demo":{"digest":"d1"}`) || !strings.Contains(string(wire), `"raw":{"module":"raw","version":"1"}`) { + t.Fatalf("on the wire: %s", wire) + } + back, err := Decode(wire) + if err != nil { + t.Fatal(err) + } + if _, err := Run(back); !errors.Is(err, ErrUnfilled) { + t.Fatalf("a digest left unfilled runs: %v", err) + } + asked := 0 + if err := Fill(&back, func(d string) ([]byte, error) { + asked++ + if d != "d1" { + return nil, errors.New("no such digest") + } + return []byte(demo), nil + }); err != nil { + t.Fatal(err) + } + if asked != 2 || string(back.Cases[0].Manifests["demo"].Raw) != demo || string(back.Cases[0].Kept["laptop"]["demo"].Raw) != demo { + t.Fatalf("filled %d times: %+v", asked, back.Cases[0]) + } + bad := Request{V: Version, Cases: []Case{{Name: "c", Manifests: map[string]Manifest{"x": {Digest: "nope"}}}}} + if err := Fill(&bad, func(string) ([]byte, error) { return nil, errors.New("not there") }); err == nil || !strings.Contains(err.Error(), "nope") { + t.Fatalf("a digest the indexer does not hold: %v", err) + } +} + +// The input travels as JSON and comes back equal, so the command composes what its caller meant. +func TestTheInputTravelsAsJSON(t *testing.T) { + in := compose.Input{ + Nodes: []compose.Node{{Name: "anchor", Assigned: []string{"nats"}, Capabilities: map[string]bool{"systemd": true}, + Place: compose.Place{Endpoint: "anchor.example:51820", Site: "here", Hub: true, Address: "10.77.0.1"}, + Pins: map[string]catalogue.Chosen{"bus": {Node: "anchor", Module: "nats"}}, + Settings: map[string][]catalogue.Layer{"nats": {{Values: map[string]any{"port": float64(4222)}}}}}}, + Holdings: []catalogue.Held{{Claim: "mesh-broker", Scope: "mesh", Node: "anchor", Module: "nats"}}, + Issued: []compose.Issued{{Provision: "bus", Consumer: "laptop", ConsumerModule: "x", Provider: "anchor"}}, + } + raw, err := json.Marshal(Request{V: Version, Input: in}) + if err != nil { + t.Fatal(err) + } + back, err := Decode(raw) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(back.Input, in) { + t.Fatalf("the input came back as\n%+v\nwant\n%+v", back.Input, in) + } +} + +// The seats a node's assigned modules claim are held there when the request says the holdings come from the claims. +func TestHoldingsComeFromTheClaims(t *testing.T) { + m, err := catalogue.ParseManifest([]byte(demo)) + if err != nil { + t.Fatal(err) + } + got := HoldingsOf(map[string]catalogue.Manifest{"demo": m}, []compose.Node{{Name: "b", Assigned: []string{"demo"}}, {Name: "a", Assigned: []string{"demo"}}}) + if len(got) != 2 || got[0].Node != "a" || got[0].Claim != "node-files" || got[0].Scope != "node" { + t.Fatalf("holdings = %+v", got) + } +} + +// A request the command cannot run is refused, never answered as a verdict: another version, a case named twice, an +// unknown depth. +func TestARequestItCannotRunIsRefused(t *testing.T) { + if _, err := Decode([]byte(`{"v": 2}`)); err == nil { + t.Error("a request of version 2 is read") + } + if _, err := Run(Request{V: Version, Cases: []Case{{Name: "a"}, {Name: "a"}}}); err == nil { + t.Error("a case named twice runs") + } + if _, err := Run(Request{V: Version, Depth: "deep"}); err == nil { + t.Error("an unknown depth runs") + } + v, err := Run(Request{V: Version, Cases: []Case{{Name: "only-read", Manifests: map[string]Manifest{"demo": {Raw: json.RawMessage(demo)}}}}}) + if err != nil || len(v.Cases) != 1 || !v.Cases[0].Manifests["demo"].Accepted || v.Cases[0].Machines != nil || v.SDK == "" { + t.Fatalf("a request with no machines: %+v, %v", v, err) + } +} + +// A manifest whose config value falls back to a predefined key (novox/hq ADR 0348 §7) is read, not refused for the +// list of predefined keys at the catalogue's root that a run is never handed with its manifests: the merge gate judges +// the fallback in the catalogue. xterm's module.json from mesh-catalog at 835bf22b falls back to "font-size"; a group +// touching xterm or pacman failed its composition on this (group check 18df81380db3). A list in the request is the one +// judged against, so a fallback it does not name is still refused. +func TestAManifestFallingBackToAPredefinedKeyIsRead(t *testing.T) { + raw, err := os.ReadFile("../catalogue/testdata/catalogue-835bf22b-xterm.module.json") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(raw), `"predefined"`) { + t.Fatal("the fixture names no predefined key") + } + r := Request{V: Version, Cases: []Case{{Name: "change", Manifests: map[string]Manifest{"xterm": {Raw: raw}}}}} + v, err := Run(r) + if err != nil { + t.Fatal(err) + } + if got := v.Cases[0].Manifests["xterm"]; !got.Accepted { + t.Fatalf("xterm: %+v, want accepted", got) + } + r.Predefined = catalogue.PredefinedList{"something-else": {Type: "string", Description: "x"}} + if v, err = Run(r); err != nil { + t.Fatal(err) + } + if got := v.Cases[0].Manifests["xterm"]; got.Accepted || !strings.Contains(got.Refused, "not on the list of predefined keys") { + t.Fatalf("xterm judged against a list without its key: %+v, want refused naming the list", got) + } +} diff --git a/go/delivery/composed.go b/go/delivery/composed.go index 0509bbb..fa771cf 100644 --- a/go/delivery/composed.go +++ b/go/delivery/composed.go @@ -14,7 +14,7 @@ import ( // resolver, identities and groups) under `.node`. Beside it, under the same key in `mesh-controller_composed`, // which the assigner reads, the controller writes the composition's digest and its judgement of the move from the // composition the node runs now (ComposedDigest). The controller writes both by compare-and-set while it holds the -// lease; the operator's credential is a second writer on the hand path alone (clause 10, mesh-compose). +// lease; the operator's credential is a second writer on the hand path alone (clause 10, mesh-hand-compose). // // A composition is derived state (clause 4): its key and its digest are deleted by the controller when its fingerprint // is no longer among the module's five newest and no want names it, and each bucket keeps five revisions of a key. @@ -143,7 +143,7 @@ func checkDigest(digest, of string) error { // Policy is the module's upgrade policy as the controller holds it (ADR 0236, ADR 0539 clause 14), with PolicyFrom // where it comes from and PolicyWhy why: the assigner holds a move of a module of policy record from it, the operator's // own choice (`upgrade` with policy) included. Rewritten when the policy changes. Empty on a composition made by hand -// (mesh-compose), whose want the operator writes. +// (mesh-hand-compose), whose want the operator writes. type ComposedDigest struct { V int `json:"v"` Digest string `json:"digest"` diff --git a/go/delivery/composed_signature.go b/go/delivery/composed_signature.go index 1860a5a..ad3d042 100644 --- a/go/delivery/composed_signature.go +++ b/go/delivery/composed_signature.go @@ -7,7 +7,7 @@ import ( ) // A composition's signature (novox/hq ADR 0539 clause 3): the controller signs each composition it writes with its -// key, and the operator signs one made by hand with mesh-compose with theirs. The node-engine trusts exactly these two +// key, and the operator signs one made by hand with mesh-hand-compose with theirs. The node-engine trusts exactly these two // signers and applies nothing of a composition whose signature fails or whose signer is neither. The signature is an // ed25519 signature, as a declaration's is, of the composition's digest (ComposedDigestOf) after a context line, so a // signature made for anything else never verifies here. diff --git a/go/delivery/stepasks.go b/go/delivery/stepasks.go index 1e8cdd3..134dae8 100644 --- a/go/delivery/stepasks.go +++ b/go/delivery/stepasks.go @@ -1,8 +1,11 @@ package delivery import ( + "crypto/sha256" + "encoding/hex" "encoding/json" "fmt" + "path" "regexp" "sort" "strings" @@ -154,8 +157,8 @@ func checkAskID(id string) error { } // StepAsk is the message on a build seat's stream: who asked what, of which kind and language, and the one whole job. -// Exactly one of Build and Check is set: a delivery ask is a build; a check ask a build (a candidate's fingerprint) or a -// check; a recheck ask a check; an image ask is always a build. +// Exactly one of Build, Check and Compose is set: a delivery ask is a build; a check ask a build (a candidate's +// fingerprint), a check or, on the Go seat, a compose; a recheck ask a check; an image ask is always a build. type StepAsk struct { V int `json:"v"` ID string `json:"id"` @@ -163,10 +166,48 @@ type StepAsk struct { Language string `json:"language"` // Asker is the module that put the ask (mesh-delivery-builder or mesh-delivery-checker); the stream's grant is what // makes it true, this field only names it to a reader. - Asker string `json:"asker"` - At time.Time `json:"at"` - Build *BuildJob `json:"build,omitempty"` - Check *CheckJob `json:"check,omitempty"` + Asker string `json:"asker"` + At time.Time `json:"at"` + Build *BuildJob `json:"build,omitempty"` + Check *CheckJob `json:"check,omitempty"` + Compose *ComposeJob `json:"compose,omitempty"` +} + +// ComposeJob is a compose ask's job (novox/hq ADR 0482 clause 4, issue 1622): fetch the commit, build the composer +// command (Package) in the Go module at Dir of the fetched tree, once per ComposerKey, run it in a run step shut off +// with Request on its standard input, and answer its standard output whole (ComposeAnswer). The assigner asks it, to +// compose a group with its mesh-sdk member's head, and to read the group's manifests with the mesh-sdk the running +// controller and node-engine pin (ADR 0482 clause 5, issue 1623). +type ComposeJob struct { + // Fetch is the fetch tool's arguments: the repository and the full commit whose mesh-sdk composes (mesh-sdk itself + // at a head, or a module that requires mesh-sdk, at the version it pins). + Fetch map[string]any `json:"fetch"` + // Dir is the directory of the Go module the composer is built in, relative to the repository's root ("go" for + // mesh-sdk, RootPath for a module at the root); Package is the composer's import path (mesh-sdk composer.Package). + Dir string `json:"dir"` + Package string `json:"package"` + // Request is the composer's request (mesh-sdk composer.Request), its manifests named by their digests in the + // indexer's ManifestsBucket: the build module fills them before the run (composer.Fill). + Request json.RawMessage `json:"request"` + // Reader names, for a reader of the ask, whose composition this is: the mesh-sdk head of a group, or a module whose + // mesh-sdk reads the group's manifests (mesh-controller, mesh-host). + Reader string `json:"reader,omitempty"` +} + +// ComposeAnswer is a compose ask's step result: whether this run built the composer or took the one kept under its +// key, the key, and the composer's verdict (mesh-sdk composer.Verdict) whole. +type ComposeAnswer struct { + Built bool `json:"built"` + Key string `json:"key"` + Verdict json.RawMessage `json:"verdict"` +} + +// ComposerKey is the key a build module keeps a built composer under: the repository, commit, directory and package it +// was built from and the toolchain it was built with, hashed (`composer-` and 16 hex digits), so each head is built once +// on a node (ADR 0482 clause 4). +func ComposerKey(repository, commit, dir, pkg string, toolchain ToolchainUsed) string { + sum := sha256.Sum256([]byte(strings.Join([]string{repository, commit, dir, pkg, toolchain.Version, toolchain.Recipe}, "\n"))) + return "composer-" + hex.EncodeToString(sum[:8]) } // BuildJob is a build ask's job: fetch the commit (unless every artifact is a copy), then build or copy each artifact, @@ -232,9 +273,17 @@ func (a StepAsk) check() error { if a.At.IsZero() { return fmt.Errorf("delivery: the ask %s names no time", a.ID) } + jobs := 0 + for _, set := range []bool{a.Build != nil, a.Check != nil, a.Compose != nil} { + if set { + jobs++ + } + } switch { - case (a.Build == nil) == (a.Check == nil): - return fmt.Errorf("delivery: the ask %s holds a build or a check, exactly one", a.ID) + case jobs != 1: + return fmt.Errorf("delivery: the ask %s holds a build, a check or a compose, exactly one", a.ID) + case a.Compose != nil && (a.Kind != AskCheck || a.Language != AskGo): + return fmt.Errorf("delivery: the ask %s is a compose, which the Go seat's check stream alone carries", a.ID) case a.Build != nil && a.Kind == AskRecheck: return fmt.Errorf("delivery: the ask %s is a build on the recheck stream, which carries checks alone", a.ID) case a.Check != nil && a.Kind == AskDelivery: @@ -245,9 +294,28 @@ func (a StepAsk) check() error { if a.Build != nil { return a.Build.check(a.ID) } + if a.Compose != nil { + return a.Compose.check(a.ID) + } return a.Check.check(a.ID) } +func (c ComposeJob) check(id string) error { + if err := checkFetchArgs(id, c.Fetch); err != nil { + return err + } + if c.Dir == "" || c.Dir != path.Clean(c.Dir) || strings.HasPrefix(c.Dir, "/") || c.Dir == ".." || strings.HasPrefix(c.Dir, "../") { + return fmt.Errorf("delivery: the compose ask %s names no clean directory inside the repository (%q)", id, c.Dir) + } + if c.Package == "" { + return fmt.Errorf("delivery: the compose ask %s names no package to build", id) + } + if !json.Valid(c.Request) || len(c.Request) == 0 || c.Request[0] != '{' { + return fmt.Errorf("delivery: the compose ask %s holds no request object", id) + } + return nil +} + func (b BuildJob) check(id string) error { if err := checkName("module", b.Module); err != nil { return err diff --git a/go/delivery/stepasks_compose_test.go b/go/delivery/stepasks_compose_test.go new file mode 100644 index 0000000..a9547f0 --- /dev/null +++ b/go/delivery/stepasks_compose_test.go @@ -0,0 +1,76 @@ +package delivery + +import ( + "encoding/json" + "reflect" + "strings" + "testing" + "time" +) + +func composeAsk() StepAsk { + return StepAsk{ID: "mh3k2x9qa1b2c5", Kind: AskCheck, Language: AskGo, Asker: "mesh-delivery-assigner", + At: time.Date(2026, 10, 24, 3, 0, 0, 0, time.UTC), + Compose: &ComposeJob{Fetch: map[string]any{"repository": "novox/mesh-sdk", "commit": askCommit}, + Dir: "go", Package: "git.novox.be/novox/mesh-sdk/go/cmd/mesh-compose", + Request: json.RawMessage(`{"v":1,"cases":[{"name":"change","manifests":{"demo":{"digest":"d1"}}}]}`), Reader: "mesh-sdk head"}} +} + +// 1622: a compose ask encodes and decodes with every field. +func TestAComposeAskRoundTrips(t *testing.T) { + a := composeAsk() + raw, err := EncodeStepAsk(a) + if err != nil { + t.Fatal(err) + } + got, err := DecodeStepAsk(raw) + if err != nil { + t.Fatal(err) + } + a.V = StepAskVersion + if !reflect.DeepEqual(got, a) { + t.Fatalf("round trip:\n got %+v\nwant %+v", got, a) + } +} + +// 1622: a compose ask goes only on the Go seat's check stream, holds no other job beside it, fetches a full commit, +// builds a package in a clean directory of the repository and carries a request object. +func TestAComposeAskIsRefusedWhereItDoesNotBelong(t *testing.T) { + for name, change := range map[string]func(*StepAsk){ + "on the delivery stream": func(a *StepAsk) { a.Kind = AskDelivery }, + "on the recheck stream": func(a *StepAsk) { a.Kind = AskRecheck }, + "on the Node seat": func(a *StepAsk) { a.Language = AskNode }, + "beside a check": func(a *StepAsk) { a.Check = checkAsk().Check }, + "with a short commit": func(a *StepAsk) { a.Compose.Fetch["commit"] = "0123456" }, + "in no directory": func(a *StepAsk) { a.Compose.Dir = "" }, + "outside the repository": func(a *StepAsk) { a.Compose.Dir = "../x" }, + "with no package": func(a *StepAsk) { a.Compose.Package = "" }, + "with no request": func(a *StepAsk) { a.Compose.Request = nil }, + "with a request not JSON": func(a *StepAsk) { a.Compose.Request = json.RawMessage(`[1]`) }, + } { + a := composeAsk() + change(&a) + if _, err := EncodeStepAsk(a); err == nil { + t.Errorf("a compose ask %s is encoded", name) + } + } +} + +// 1622/AC3: a composer is kept under one key per repository, commit, directory, package and toolchain, so two asks of +// one head build it once on a node, and another head or toolchain builds its own. +func TestAComposerIsKeptByWhatItWasBuiltFrom(t *testing.T) { + tc := ToolchainUsed{Version: "go1.26.0", Recipe: "3"} + k := ComposerKey("novox/mesh-sdk", askCommit, "go", "p", tc) + if k != ComposerKey("novox/mesh-sdk", askCommit, "go", "p", tc) || !strings.HasPrefix(k, "composer-") || len(k) != len("composer-")+16 { + t.Fatalf("key = %q", k) + } + for _, other := range []string{ + ComposerKey("novox/mesh-sdk", strings.Repeat("1", 40), "go", "p", tc), + ComposerKey("novox/mesh-controller", askCommit, ".", "p", tc), + ComposerKey("novox/mesh-sdk", askCommit, "go", "p", ToolchainUsed{Version: "go1.26.1", Recipe: "3"}), + } { + if other == k { + t.Errorf("two composers built from different inputs share the key %s", k) + } + } +} diff --git a/go/internal/beside/beside.go b/go/internal/beside/beside.go index 93a0d40..d31ec1d 100644 --- a/go/internal/beside/beside.go +++ b/go/internal/beside/beside.go @@ -15,7 +15,12 @@ // `mesh/repo-check` runs; a repository missing there fails the test, never skips it. Which clones a // check gets is chosen from the inventory: mesh-catalog by the source of the `nats` module, mesh-host // by the source of `mesh-host`, and mesh-sdk as the controller's sibling at the commit the controller's -// go.mod pins. In a mesh where either module has no source repository nothing is +// go.mod pins. Each repository's tests read these siblings, and the checker places exactly these +// (mesh-delivery's checker, besideRules, follows this list): mesh-controller's read mesh-catalog, +// mesh-host and mesh-sdk; mesh-host's read mesh-catalog, mesh-sdk and mesh-controller (its step-nine +// test reads the controller's own manifest, novox/hq issue 1295); mesh-sdk's read mesh-catalog and +// mesh-host; mesh-catalog's read mesh-sdk and mesh-controller; mesh-delivery's read mesh-sdk and +// mesh-catalog. In a mesh where either module has no source repository nothing is // cloned, and these tests fail loudly with "not beside this check": a cause in the setup, not in the // change. And in a delivery group that holds a mesh-catalog pull request, these tests read the // catalogue's main, not the group's head. diff --git a/go/internal/beside/reads-outside b/go/internal/beside/reads-outside new file mode 100644 index 0000000..f6b2d87 --- /dev/null +++ b/go/internal/beside/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/go/packageguard/reads-outside b/go/packageguard/reads-outside new file mode 100644 index 0000000..f6b2d87 --- /dev/null +++ b/go/packageguard/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/go/reads-outside b/go/reads-outside new file mode 100644 index 0000000..f6b2d87 --- /dev/null +++ b/go/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/go/sdkversion/sdkversion.go b/go/sdkversion/sdkversion.go index 46a705a..3cbb82e 100644 --- a/go/sdkversion/sdkversion.go +++ b/go/sdkversion/sdkversion.go @@ -24,7 +24,11 @@ func Version() string { // FromBuildInfo reads the SDK's version out of a build info. A dependency replaced by another module version is that // version; replaced by a local path it is the path, never empty. When the SDK is the main module it is "(devel)", -// followed by a space and the vcs.revision when the build stamped one. Without the SDK in the build it is "unknown". +// followed by a space and the vcs.revision when the build stamped one. An SDK command built from another module that +// requires the SDK (`go build git.novox.be/novox/mesh-sdk/go/cmd/mesh-compose` in a reader's tree, as mesh-build-go's +// compose job builds it: novox/hq ADR 0482 clause 5) has the SDK as its main module at the version that module pins, +// read from the module cache with no vcs stamp: it is that version, never a bare "(devel)" that names no SDK. Without +// the SDK in the build it is "unknown". func FromBuildInfo(bi *debug.BuildInfo) string { if bi == nil { return "unknown" @@ -35,6 +39,9 @@ func FromBuildInfo(bi *debug.BuildInfo) string { return "(devel) " + s.Value } } + if v := bi.Main.Version; v != "" && v != "(devel)" { + return v + } return "(devel)" } for _, d := range bi.Deps { diff --git a/go/sdkversion/sdkversion_test.go b/go/sdkversion/sdkversion_test.go index 7a4b04d..3cc60a2 100644 --- a/go/sdkversion/sdkversion_test.go +++ b/go/sdkversion/sdkversion_test.go @@ -42,6 +42,18 @@ func TestTheMainModuleIsDevelWithItsRevision(t *testing.T) { } } +// The shape go gives `go build ` run in a module that requires the SDK (captured with go1.27 from +// mesh-compose built in a reader's tree at bc0c25e51052: "mod git.novox.be/novox/mesh-sdk/go v0.1.14-…-bc0c25e51052", +// no vcs settings): the SDK is the main module at the pinned version, which names the SDK the binary carries. +func TestAnSDKCommandBuiltFromAReaderIsThePinnedVersion(t *testing.T) { + bi := &debug.BuildInfo{Path: sdkPath + "/cmd/mesh-compose", + Main: debug.Module{Path: sdkPath, Version: "v0.1.14-0.20261024112544-bc0c25e51052"}, + Settings: []debug.BuildSetting{{Key: "-trimpath", Value: "true"}, {Key: "CGO_ENABLED", Value: "0"}}} + if got := FromBuildInfo(bi); got != "v0.1.14-0.20261024112544-bc0c25e51052" { + t.Fatalf("got %q", got) + } +} + func TestNoBuildInfoIsUnknown(t *testing.T) { if got := FromBuildInfo(nil); got != "unknown" { t.Fatalf("got %q", got)