Merge main (the R4g finish: the composer command) into feat/r6 (issue 1943)
mesh/delivery delivering: 0 machine step(s) passed
mesh/delivery-group group feat/r6 merged: every member is merged
mesh/check failed: the composition: the composer of the mesh-sdk mesh-controller at bdb8193456e17c9df59215c7f3e69a83a2e5a262 pins did not run: failed …

main's mesh-compose is the composer command (ADR 0482 clause 4) and keeps its name, which composer.Package and the
build steps name; feat/r6's hand path tool, which composes one module for one node and signs it with the operator's key
(ADR 0539 clause 10), moves to cmd/mesh-hand-compose.
This commit is contained in:
2026-10-24 23:01:18 +02:00
21 changed files with 1463 additions and 299 deletions
+2 -5
View File
@@ -127,11 +127,8 @@ func TestPullHoldsAtMostItsCapacity(t *testing.T) {
} }
release <- struct{}{} release <- struct{}{}
waitFor(t, "the third taken", func() bool { return src.Waiting(delivery.AskDelivery) == 0 }) waitFor(t, "the third taken", func() bool { return src.Waiting(delivery.AskDelivery) == 0 })
mu.Lock() // Taken from the source comes a moment before its run starts: wait for the start, not only the take.
if started != 3 { waitFor(t, "the third started", func() bool { mu.Lock(); defer mu.Unlock(); return started == 3 })
t.Errorf("started %d, want 3", started)
}
mu.Unlock()
close(release) close(release)
waitFor(t, "all acked", func() bool { a, _, _ := src.Fates(); return len(a) == 3 }) waitFor(t, "all acked", func() bool { a, _, _ := src.Fates(); return len(a) == 3 })
} }
+1
View File
@@ -0,0 +1 @@
These tests read files outside this directory (the repository's root files or other directories): mesh-build-go never keeps this package's verdict (issue 1935).
+1 -1
View File
@@ -81,7 +81,7 @@ var RemovedControllerVerbs = map[string]string{
"plans": "the delivery view (nox-delivery): each machine's want and have, read from the assigner's wants and the " + "plans": "the delivery view (nox-delivery): each machine's want and have, read from the assigner's wants and the " +
"node-engine's haves", "node-engine's haves",
"plan": "the node's compositions: mesh-controller_composed holds each module's digest and judgement per node, and " + "plan": "the node's compositions: mesh-controller_composed holds each module's digest and judgement per node, and " +
"mesh-compose prints one module's composition for one node", "mesh-hand-compose prints one module's composition for one node",
"push": "a want: the assigner writes want.<node>.<module> by its rules, and by hand the operator's credential writes " + "push": "a want: the assigner writes want.<node>.<module> by its rules, and by hand the operator's credential writes " +
"it, naming a kept composition (ADR 0539 clause 10)", "it, naming a kept composition (ADR 0539 clause 10)",
"deliver": "nothing to start: a merged change reaches each machine by its want", "deliver": "nothing to start: a merged change reaches each machine by its want",
+123 -169
View File
@@ -1,209 +1,163 @@
// mesh-compose composes one module for one node by hand, from the inputs the controller keeps, and signs it with the // mesh-compose is mesh-sdk's composer command (novox/hq ADR 0482 clause 4, issue 1621): it reads a composer.Request on
// operator's key, for the hand path of novox/hq ADR 0539 clause 10: a fingerprint the controller never composed (a // standard input, validates every manifest with this mesh-sdk's reader, composes the machines named in each case and
// hand build), while the controller cannot compose its own fix. It composes with mesh-sdk's composition, the one the // writes one composer.Verdict on standard output. Built at any commit of mesh-sdk (or at the mesh-sdk a module pins),
// controller writes with (compose.Compositions), and prints the composition, its key and its digest; only with -write // it answers what that commit's composition answers, so a program that links one mesh-sdk can have another's judged.
// does it write them, with the operator's credential: the composition into the node's bucket composed-<node>, each
// file over 64 KiB into the node's object bucket, and the digest into mesh-controller_composed. The want is the
// operator's next act, naming that digest; each act is recorded in the hand-act log.
// //
// mesh-compose -input <inputs.json> -node <node> -module <module> -fp <src2:...> -key <operator key> // mesh-compose < request.json > verdict.json
// [-write -nats <url> -creds <operator credential>] // mesh-compose -request request.json > verdict.json
// mesh-compose -catalogue <dir> [-node <name>]... [-depth resolve|compose] [-declarations] [< request.json]
// //
// The inputs are a compose.Input as JSON. The key is the operator's ed25519 private key, PKCS#8 in PEM; the // -catalogue reads <dir>/modules/*/module.json and a module.json at <dir>'s root as the first case's manifests (a
// node-engine trusts it beside the controller's (ADR 0539 clause 3). // checkout or `git archive` of the catalogue repository at a ref: the command fetches nothing). -node adds a machine to
// compose; -declarations answers each machine's declared paths with their owners and modes.
//
// It opens no network connection, bus, store or file beyond its standard input, the -request file and the -catalogue
// directory, and reads
// no environment. Exit codes: 0 the verdict is written (a machine that does not compose is a verdict, not an error);
// 1 the request cannot be read or run, said on standard error; 2 the flags are wrong.
package main package main
import ( import (
"context"
"crypto/ed25519"
"crypto/x509"
"encoding/json" "encoding/json"
"encoding/pem"
"errors" "errors"
"flag" "flag"
"fmt" "fmt"
"io" "io"
"os" "os"
"time" "path/filepath"
"sort"
"strings"
"github.com/nats-io/nats.go" "git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-sdk/go/composer"
"git.novox.be/novox/mesh-sdk/go/compose"
"git.novox.be/novox/mesh-sdk/go/declaration"
"git.novox.be/novox/mesh-sdk/go/delivery"
) )
func main() { func main() {
if err := run(os.Args[1:], os.Stdout); err != nil { os.Exit(run(os.Args[1:], os.Stdin, os.Stdout, os.Stderr))
fmt.Fprintln(os.Stderr, "mesh-compose:", err)
os.Exit(1)
}
} }
// printed is what mesh-compose prints: where the composition goes, its digest, the composition, and whether it was type nodes []string
// written.
type printed struct {
Bucket string `json:"bucket"`
Key string `json:"key"`
Digest string `json:"digest"`
Composition json.RawMessage `json:"composition"`
Objects []string `json:"objects,omitempty"`
Written bool `json:"written"`
Next string `json:"next"`
}
func run(args []string, stdout io.Writer) error { func (n *nodes) String() string { return strings.Join(*n, ",") }
func (n *nodes) Set(v string) error { *n = append(*n, v); return nil }
// run is the command: its arguments, standard input, output and error; it answers the exit code.
func run(args []string, stdin io.Reader, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("mesh-compose", flag.ContinueOnError) fs := flag.NewFlagSet("mesh-compose", flag.ContinueOnError)
input := fs.String("input", "", "the controller's kept inputs, a compose.Input as JSON") fs.SetOutput(stderr)
node := fs.String("node", "", "the node to compose for") catalogueDir := fs.String("catalogue", "", "a catalogue checkout: <dir>/modules/*/module.json are the first case's manifests")
module := fs.String("module", "", "the module to compose") var machines nodes
fp := fs.String("fp", "", "the module's fingerprint, src2: and 64 hex digits") fs.Var(&machines, "node", "a machine to compose (repeatable)")
keyFile := fs.String("key", "", "the operator's ed25519 private key, PKCS#8 in PEM") depth := fs.String("depth", "", "resolve or compose (default: the request's, else compose)")
write := fs.Bool("write", false, "write the composition, its files and its digest, with the operator's credential") declarations := fs.Bool("declarations", false, "answer each composed machine's declared paths, owners and modes")
url := fs.String("nats", nats.DefaultURL, "with -write: the bus") requestFile := fs.String("request", "", "read the request from this file instead of standard input (a run step that hands no standard input)")
creds := fs.String("creds", "", "with -write: the operator's credential file")
if err := fs.Parse(args); err != nil { if err := fs.Parse(args); err != nil {
return err return 2
} }
switch { if fs.NArg() > 0 {
case *input == "" || *node == "" || *module == "" || *fp == "": fmt.Fprintf(stderr, "mesh-compose: unexpected arguments %q\n", fs.Args())
return errors.New("-input, -node, -module and -fp are each required") return 2
case *keyFile == "": }
return errors.New("-key is required: a composition made by hand is signed by the operator's key, and the node-engine applies no other") var raw []byte
case *write && *creds == "": var err error
return errors.New("-write needs -creds: the operator's credential is the only hand writer of a composition") if *requestFile != "" {
raw, err = os.ReadFile(*requestFile)
} else {
raw, err = io.ReadAll(stdin)
} }
key, err := readKey(*keyFile)
if err != nil { if err != nil {
return err fmt.Fprintln(stderr, "mesh-compose: the request cannot be read:", err)
return 1
} }
key2, err := delivery.ComposedKey(*node, *module, *fp) var req composer.Request
if err != nil { if strings.TrimSpace(string(raw)) == "" {
return err if *catalogueDir == "" {
} fmt.Fprintln(stderr, "mesh-compose: no request on standard input and no -catalogue")
bucket, err := delivery.ComposedBucketFor(*node) return 1
if err != nil {
return err
}
raw, err := os.ReadFile(*input)
if err != nil {
return err
}
var in compose.Input
if err := json.Unmarshal(raw, &in); err != nil {
return fmt.Errorf("the inputs in %s are not a compose.Input: %w", *input, err)
}
all, _, err := compose.Compositions(in, *node, map[string]string{*module: *fp}, compose.Reading)
if err != nil {
return err
}
c, ok := all[key2]
if !ok {
return fmt.Errorf("%s does not run %s: its composition holds none of it", *node, *module)
}
c, objects, err := c.ByDigest()
if err != nil {
return err
}
if c, err = delivery.SignComposed(c, key); err != nil {
return err
}
encoded, err := delivery.EncodeComposed(c)
if err != nil {
return err
}
digest, err := delivery.ComposedDigestOf(c)
if err != nil {
return err
}
out := printed{Bucket: bucket, Key: key2, Digest: digest, Composition: encoded,
Next: fmt.Sprintf("write want.%s.%s naming fp %s and composed %s with the operator's credential, and record each act in the hand-act log", *node, *module, *fp, digest)}
for d := range objects {
out.Objects = append(out.Objects, d)
}
if *write {
if err := writeIt(*url, *creds, *node, bucket, key2, encoded, digest, objects); err != nil {
return err
} }
out.Written = true req = composer.Request{V: composer.Version}
} else if req, err = composer.Decode(raw); err != nil {
fmt.Fprintln(stderr, "mesh-compose:", err)
return 1
}
if *catalogueDir != "" {
read, err := readCatalogue(*catalogueDir)
if err != nil {
fmt.Fprintln(stderr, "mesh-compose:", err)
return 1
}
if len(req.Cases) == 0 {
req.Cases = []composer.Case{{Name: "catalogue", Manifests: map[string]composer.Manifest{}}}
}
if req.Cases[0].Manifests == nil {
req.Cases[0].Manifests = map[string]composer.Manifest{}
}
for name, m := range read {
if _, given := req.Cases[0].Manifests[name]; !given {
req.Cases[0].Manifests[name] = m
}
}
// The catalogue's list of predefined keys is the one its manifests' fallbacks are judged against (ADR 0348 §7).
if req.Predefined == nil {
list, err := catalogue.ReadPredefined(*catalogueDir)
if err != nil {
fmt.Fprintln(stderr, "mesh-compose:", err)
return 1
}
req.Predefined = list
}
}
req.Machines = append(req.Machines, machines...)
if *depth != "" {
req.Depth = *depth
}
if *declarations {
req.Declarations = true
}
v, err := composer.Run(req)
if err != nil {
fmt.Fprintln(stderr, "mesh-compose:", err)
return 1
} }
enc := json.NewEncoder(stdout) enc := json.NewEncoder(stdout)
enc.SetIndent("", " ") if err := enc.Encode(v); err != nil {
return enc.Encode(out) fmt.Fprintln(stderr, "mesh-compose: the verdict cannot be written:", err)
return 1
}
return 0
} }
func readKey(path string) (ed25519.PrivateKey, error) { // readCatalogue reads every module.json of a catalogue checkout by its module's name: the root's and each of
raw, err := os.ReadFile(path) // modules/*/. A manifest whose name cannot be read is kept under its directory's name, so the reader's refusal is said.
func readCatalogue(dir string) (map[string]composer.Manifest, error) {
paths, err := filepath.Glob(filepath.Join(dir, "modules", "*", "module.json"))
if err != nil { if err != nil {
return nil, err return nil, err
} }
block, _ := pem.Decode(raw) if _, err := os.Stat(filepath.Join(dir, "module.json")); err == nil {
if block == nil { paths = append(paths, filepath.Join(dir, "module.json"))
return nil, fmt.Errorf("%s holds no PEM block", path)
} }
parsed, err := x509.ParsePKCS8PrivateKey(block.Bytes) if len(paths) == 0 {
if err != nil { return nil, fmt.Errorf("the catalogue %s holds no module.json", dir)
return nil, fmt.Errorf("%s: %w", path, err)
} }
key, ok := parsed.(ed25519.PrivateKey) sort.Strings(paths)
if !ok { out := map[string]composer.Manifest{}
return nil, fmt.Errorf("%s is not an ed25519 key", path) for _, p := range paths {
} raw, err := os.ReadFile(p)
return key, nil
}
// writeIt writes the files by digest, then the composition, then its digest: a reader that finds the digest finds the
// composition and its files already there.
func writeIt(url, creds, node, bucket, key string, encoded []byte, digest string, objects map[string][]byte) error {
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()
nc, err := nats.Connect(url, nats.UserCredentials(creds), nats.Name("mesh-compose"))
if err != nil {
return err
}
defer nc.Close()
js, err := jetstream.New(nc)
if err != nil {
return err
}
if len(objects) > 0 {
filesBucket, err := declaration.FileBucketFor(node)
if err != nil { if err != nil {
return err return nil, err
} }
store, err := js.ObjectStore(ctx, filesBucket) var probe struct {
if err != nil { Module string `json:"module"`
return fmt.Errorf("the node's object bucket %s: %w", filesBucket, err)
} }
for d, body := range objects { name := filepath.Base(filepath.Dir(p))
name, err := declaration.ObjectName(d) if json.Unmarshal(raw, &probe) == nil && probe.Module != "" {
if err != nil { name = probe.Module
return err
}
if _, err := store.PutBytes(ctx, name, body); err != nil {
return fmt.Errorf("the file %s: %w", d, err)
}
} }
if _, twice := out[name]; twice {
return nil, errors.New("the catalogue " + dir + " defines " + name + " twice")
}
out[name] = composer.Manifest{Raw: raw}
} }
composed, err := js.KeyValue(ctx, bucket) return out, nil
if err != nil {
return fmt.Errorf("the bucket %s: %w", bucket, err)
}
if _, err := composed.Put(ctx, key, encoded); err != nil {
return fmt.Errorf("%s in %s: %w", key, bucket, err)
}
value, err := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digest, Judgement: delivery.JudgedCannot,
Words: "composed by hand with mesh-compose (ADR 0539 clause 10): the operator writes the want"})
if err != nil {
return err
}
digests, err := js.KeyValue(ctx, delivery.ControllerComposedBucket)
if err != nil {
return fmt.Errorf("the bucket %s: %w", delivery.ControllerComposedBucket, err)
}
if _, err := digests.Put(ctx, key, value); err != nil {
return fmt.Errorf("%s in %s: %w", key, delivery.ControllerComposedBucket, err)
}
return nil
} }
+312 -110
View File
@@ -2,144 +2,346 @@ package main
import ( import (
"bytes" "bytes"
"crypto/ed25519"
"crypto/rand"
"crypto/x509"
"encoding/base64"
"encoding/json" "encoding/json"
"encoding/pem" "go/build"
"os" "os"
"os/exec"
"path/filepath" "path/filepath"
"reflect"
"runtime"
"sort" "sort"
"strings" "strings"
"testing" "testing"
"git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/compose" "git.novox.be/novox/mesh-sdk/go/compose"
"git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/composer"
"git.novox.be/novox/mesh-sdk/go/overlay" "git.novox.be/novox/mesh-sdk/go/internal/beside"
) )
// aMesh is the Input of mesh-controller's aMesh, as mesh-sdk's compose tests make it (compose/fixture_test.go): two // captured is every manifest of the catalogue captured in testdata/beside (internal/beside), raw, by module name.
// machines on the private network, the module the mesh provides for itself assigned to both. The manifest is the one func captured(t *testing.T) map[string]composer.Manifest {
// the controller provides (overlay.Manifest), read back as the catalogue reads it.
func aMesh(t *testing.T) compose.Input {
t.Helper() t.Helper()
var m catalogue.Manifest read, err := readCatalogue(filepath.Dir(beside.Catalogue(t)))
b, _ := json.Marshal(overlay.Manifest()) if err != nil {
if err := json.Unmarshal(b, &m); err != nil {
t.Fatal(err) t.Fatal(err)
} }
in := compose.Input{Catalogue: map[string]catalogue.Manifest{overlay.Name: m}} if len(read) < 100 {
for i, name := range []string{"anchor", "laptop"} { t.Fatalf("the captured catalogue holds %d manifests, fewer than the catalogue's", len(read))
pub, _, _ := ed25519.GenerateKey(rand.Reader)
in.Nodes = append(in.Nodes, compose.Node{Name: name,
Place: compose.Place{Key: base64.StdEncoding.EncodeToString(pub), Endpoint: name + ".example:51820", Site: "here", Hub: name == "anchor", Address: []string{"10.77.0.1", "10.77.0.2"}[i]},
Assigned: []string{overlay.Name}, Capabilities: map[string]bool{"container-runtime": true, "wireguard": true, "systemd": true}})
} }
return in return read
} }
func writeKey(t *testing.T, dir string) ed25519.PublicKey { // meshOf is a mesh of three machines over the captured catalogue, each assigned real modules, the input the assigner
// composes with (nodes and their assignments, holdings from the claims): anchor runs the bus and the store, laptop a
// desktop's modules, ace a module whose requirement is provided elsewhere.
func meshOf() compose.Input {
return compose.Input{Nodes: []compose.Node{
{Name: "anchor", Assigned: []string{"nats", "postgres"}},
{Name: "laptop", Assigned: []string{"sudo", "zsh"}},
{Name: "ace", Assigned: []string{"mesh-reads", "nats"}},
}}
}
func request(t *testing.T, depth string) composer.Request {
t.Helper() t.Helper()
pub, priv, _ := ed25519.GenerateKey(rand.Reader) return composer.Request{V: composer.Version, Input: meshOf(), Depth: depth, HoldingsFromClaims: true,
der, err := x509.MarshalPKCS8PrivateKey(priv) Machines: []string{"anchor", "laptop", "ace"},
if err != nil { Cases: []composer.Case{{Name: "trunk", Manifests: captured(t)}}}
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "operator.key"), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der}), 0o600); err != nil {
t.Fatal(err)
}
return pub
} }
// 1943/AC5: mesh-compose, given the kept inputs, a node, a module and its fingerprint, prints that module's func runJSON(t *testing.T, args []string, in []byte) (int, composer.Verdict, string) {
// composition for that node signed by the operator's key and its digest: its resources are the controller's for that t.Helper()
// module on that node (compose.Compose), and it verifies against the operator's key. Without --write it writes nothing. var out, errOut bytes.Buffer
func TestMeshComposeGivesTheControllersResourcesSignedByTheOperator(t *testing.T) { code := run(args, bytes.NewReader(in), &out, &errOut)
dir := t.TempDir() var v composer.Verdict
in := aMesh(t) if code == 0 {
raw, err := json.Marshal(in) if err := json.Unmarshal(out.Bytes(), &v); err != nil {
if err != nil { t.Fatalf("the verdict is not JSON: %v\n%s", err, out.String())
t.Fatal(err) }
}
input := filepath.Join(dir, "input.json")
if err := os.WriteFile(input, raw, 0o600); err != nil {
t.Fatal(err)
}
operator := writeKey(t, dir)
fp := delivery.SourcePrefix2 + strings.Repeat("a", 64)
var out bytes.Buffer
if err := run([]string{"-input", input, "-node", "laptop", "-module", overlay.Name, "-fp", fp,
"-key", filepath.Join(dir, "operator.key")}, &out); err != nil {
t.Fatal(err)
}
var printed struct {
Key string `json:"key"`
Bucket string `json:"bucket"`
Digest string `json:"digest"`
Composition json.RawMessage `json:"composition"`
Written bool `json:"written"`
}
if err := json.Unmarshal(out.Bytes(), &printed); err != nil {
t.Fatalf("%s: %v", out.String(), err)
}
c, err := delivery.DecodeComposed(printed.Composition)
if err != nil {
t.Fatal(err)
}
if digest, err := delivery.VerifyComposed(c, []ed25519.PublicKey{operator}); err != nil || digest != printed.Digest {
t.Errorf("the composition verifies %q, %v against the operator's key; printed %s", digest, err, printed.Digest)
}
wantKey, _ := delivery.ComposedKey("laptop", overlay.Name, fp)
if printed.Key != wantKey || printed.Bucket != "composed-laptop" || printed.Written {
t.Errorf("printed key %s in %s, written %v", printed.Key, printed.Bucket, printed.Written)
}
d, _, err := compose.Compose(in, "laptop")
if err != nil {
t.Fatal(err)
}
var want, got []string
for _, r := range d.Resources {
want = append(want, r["id"].(string))
}
for _, r := range c.Resources {
var head struct{ ID string }
_ = json.Unmarshal(r, &head)
got = append(got, head.ID)
}
sort.Strings(want)
sort.Strings(got)
if len(got) == 0 || strings.Join(got, " ") != strings.Join(want, " ") {
t.Errorf("mesh-compose gives %v, the controller's composition %v", got, want)
} }
return code, v, errOut.String()
} }
// 1943/AC5: mesh-compose refuses what it cannot compose, naming why: no key, a fingerprint not of the new form, a func encode(t *testing.T, r composer.Request) []byte {
// module the node does not run, a node the mesh does not have. t.Helper()
func TestMeshComposeRefuses(t *testing.T) { raw, err := json.Marshal(r)
if err != nil {
t.Fatal(err)
}
return raw
}
// 1621/AC1: the command reads the plain input and the manifests on standard input, validates every manifest, composes
// the machines named and writes one verdict: per machine composes or the reason, per manifest accepted or the field
// refused.
func TestTheCommandWritesOneVerdictPerManifestAndMachine(t *testing.T) {
r := request(t, composer.DepthResolve)
r.Cases[0].Manifests["broken"] = composer.Manifest{Raw: json.RawMessage(`{"module":"broken","version":"1","shares":["x"]}`)}
code, v, stderr := runJSON(t, nil, encode(t, r))
if code != 0 {
t.Fatalf("exit %d: %s", code, stderr)
}
if len(v.Cases) != 1 || v.Cases[0].Name != "trunk" || v.SDK == "" {
t.Fatalf("verdict = %+v", v)
}
c := v.Cases[0]
if got := c.Manifests["broken"]; got.Accepted || !strings.Contains(got.Refused, `"shares"`) {
t.Errorf("the manifest with an unknown field reads %+v, want refused naming the field", got)
}
if got := c.Manifests["nats"]; !got.Accepted {
t.Errorf("nats reads %+v, want accepted", got)
}
if len(c.Machines) != 3 {
t.Fatalf("machines = %v, want the three named", c.Machines)
}
for name, m := range c.Machines {
if !m.Composes && m.Reason == "" {
t.Errorf("%s does not compose and says no reason", name)
}
}
}
// 1621/AC3: over the captured catalogue and a mesh of three machines, the command's verdict equals, machine for
// machine and manifest for manifest, what the linked reader, resolution and composition answer, at both depths.
func TestTheCommandAnswersWhatTheLinkedCompositionAnswers(t *testing.T) {
for _, depth := range []string{composer.DepthResolve, composer.DepthCompose} {
r := request(t, depth)
code, v, stderr := runJSON(t, nil, encode(t, r))
if code != 0 {
t.Fatalf("%s: exit %d: %s", depth, code, stderr)
}
in := meshOf()
in.Catalogue = map[string]catalogue.Manifest{}
for name, m := range r.Cases[0].Manifests {
// A request without a list of predefined keys reads a fallback as the indexer does (Request.Predefined).
parsed, err := catalogue.ParseManifestForInputs(m.Raw)
got := v.Cases[0].Manifests[name]
if (err == nil) != got.Accepted || (err != nil && err.Error() != got.Refused) {
t.Errorf("%s: the manifest of %s: linked %v, command %+v", depth, name, err, got)
}
if err == nil {
in.Catalogue[name] = parsed
}
}
in.Holdings = composer.HoldingsOf(in.Catalogue, in.Nodes)
composed := 0
for _, node := range r.Machines {
got := v.Cases[0].Machines[node]
var want composer.Machine
if depth == composer.DepthResolve {
if _, err := compose.Resolve(in, node); err != nil {
want = composer.Machine{Reason: err.Error()}
} else {
want = composer.Machine{Composes: true}
}
} else {
d, problems, err := compose.Compose(in, node)
switch {
case err != nil:
want = composer.Machine{Reason: err.Error()}
default:
want = composer.Machine{Composes: len(problems) == 0, Problems: problems, Refusals: d.Refusals()}
if len(problems) > 0 {
want.Reason = "the node-engine's validator refuses the declaration"
}
}
}
if !reflect.DeepEqual(got, want) {
t.Errorf("%s: %s: command %+v, linked %+v", depth, node, got, want)
}
if got.Composes {
composed++
} else {
t.Logf("%s: %s does not compose: %s %v", depth, node, got.Reason, got.Problems)
}
}
t.Logf("%s: %d of %d machines compose", depth, composed, len(r.Machines))
}
}
// 1621 note 6: a person composes a catalogue checkout (repository@ref checked out or archived: the command fetches
// nothing) for a node and reads its declaration: each path with its kind, owner and mode.
func TestACatalogueCheckoutIsComposedForANode(t *testing.T) {
dir := t.TempDir() dir := t.TempDir()
raw, _ := json.Marshal(aMesh(t)) if err := os.MkdirAll(filepath.Join(dir, "modules", "demo"), 0o755); err != nil {
input := filepath.Join(dir, "input.json") t.Fatal(err)
_ = os.WriteFile(input, raw, 0o600) }
writeKey(t, dir) manifest := `{"module": "demo", "version": "1", "resources": [{"id": "state", "type": "directory", ` +
key := filepath.Join(dir, "operator.key") `"path": "/var/lib/demo", "mode": "0750", "owner": "root"}]}`
fp := delivery.SourcePrefix2 + strings.Repeat("a", 64) if err := os.WriteFile(filepath.Join(dir, "modules", "demo", "module.json"), []byte(manifest), 0o644); err != nil {
t.Fatal(err)
}
in := encode(t, composer.Request{V: composer.Version, Input: compose.Input{Nodes: []compose.Node{{Name: "laptop", Assigned: []string{"demo"}}}}})
code, v, stderr := runJSON(t, []string{"-catalogue", dir, "-node", "laptop", "-declarations"}, in)
if code != 0 {
t.Fatalf("exit %d: %s", code, stderr)
}
m := v.Cases[0].Machines["laptop"]
want := []composer.Path{{Path: "/var/lib/demo", Kind: "directory", Owner: "root", Mode: "0750", Module: ""}}
if !m.Composes || len(m.Declared) != 1 || m.Declared[0].Path != want[0].Path || m.Declared[0].Mode != "0750" ||
m.Declared[0].Owner != "root" || m.Declared[0].Kind != "directory" {
t.Fatalf("laptop = %+v, want it composed declaring %+v", m, want)
}
}
// A run step hands the command no standard input: the request is read from the file -request names, and a file that is
// not there is a request it cannot read.
func TestTheRequestIsReadFromAFile(t *testing.T) {
file := filepath.Join(t.TempDir(), "request.json")
r := composer.Request{V: composer.Version, Cases: []composer.Case{{Name: "only", Manifests: map[string]composer.Manifest{
"demo": {Raw: json.RawMessage(`{"module":"demo","version":"1"}`)}}}}}
if err := os.WriteFile(file, encode(t, r), 0o600); err != nil {
t.Fatal(err)
}
code, v, stderr := runJSON(t, []string{"-request", file}, nil)
if code != 0 || len(v.Cases) != 1 || !v.Cases[0].Manifests["demo"].Accepted {
t.Fatalf("exit %d (%s), verdict %+v", code, stderr, v)
}
if code, _, stderr := runJSON(t, []string{"-request", file + ".gone"}, nil); code != 1 || !strings.Contains(stderr, "cannot be read") {
t.Errorf("a request file that is not there: exit %d, %q", code, stderr)
}
}
// The exit codes: 1 for a request that cannot be read or run, 2 for wrong flags, each said on standard error.
func TestTheExitCodesSayWhatFailed(t *testing.T) {
unfilled := encode(t, composer.Request{V: composer.Version, Cases: []composer.Case{{Name: "c", Manifests: map[string]composer.Manifest{"x": {Digest: "abc"}}}}})
for _, c := range []struct { for _, c := range []struct {
name string
args []string args []string
why string in string
code int
says string
}{ }{
{[]string{"-input", input, "-node", "laptop", "-module", overlay.Name, "-fp", fp}, "-key"}, {"not JSON", nil, "{", 1, "not JSON"},
{[]string{"-input", input, "-node", "laptop", "-module", overlay.Name, "-fp", "src2:1", "-key", key}, "fingerprint"}, {"another version", nil, `{"v":2}`, 1, "version 2"},
{[]string{"-input", input, "-node", "laptop", "-module", "mesh-mail", "-fp", fp, "-key", key}, "does not run mesh-mail"}, {"a digest left unfilled", nil, string(unfilled), 1, "digest"},
{[]string{"-input", input, "-node", "nowhere", "-module", overlay.Name, "-fp", fp, "-key", key}, "nowhere"}, {"an unknown depth", nil, `{"v":1,"depth":"deep"}`, 1, "depth"},
{[]string{"-input", input, "-node", "laptop", "-module", overlay.Name, "-fp", fp, "-key", key, "-write"}, "-creds"}, {"nothing to read", nil, "", 1, "no request"},
{"a wrong flag", []string{"-nope"}, "", 2, "nope"},
{"an argument", []string{"extra"}, "", 2, "unexpected"},
} { } {
var out bytes.Buffer var out, errOut bytes.Buffer
if err := run(c.args, &out); err == nil || !strings.Contains(err.Error(), c.why) { if got := run(c.args, strings.NewReader(c.in), &out, &errOut); got != c.code || !strings.Contains(errOut.String(), c.says) {
t.Errorf("mesh-compose %v = %v, want a refusal saying %q", c.args, err, c.why) t.Errorf("%s: exit %d saying %q, want %d saying %q", c.name, got, errOut.String(), c.code, c.says)
}
}
}
// sdkPackages is every package of this module the command imports, directly or not, with the packages each imports.
func sdkPackages(t *testing.T) map[string][]string {
t.Helper()
_, file, _, _ := runtime.Caller(0)
root := filepath.Join(filepath.Dir(file), "..", "..")
const module = "git.novox.be/novox/mesh-sdk/go"
out := map[string][]string{}
var walk func(path string)
walk = func(path string) {
if _, done := out[path]; done {
return
}
rel := strings.TrimPrefix(strings.TrimPrefix(path, module), "/")
pkg, err := build.Default.ImportDir(filepath.Join(root, rel), 0)
if err != nil {
t.Fatalf("%s: %v", path, err)
}
out[path] = pkg.Imports
for _, imp := range pkg.Imports {
if strings.HasPrefix(imp, module+"/") {
walk(imp)
}
}
}
walk(composer.Package)
return out
}
// 1621/AC2 and AC4: the command imports, through every package of mesh-sdk it reaches, only the standard library and
// mesh-sdk's own packages (so `go build` from mesh-sdk's module needs nothing beyond its lock), and none that reaches
// a bus, a store, a network service or another process.
func TestTheCommandImportsNoBusStoreNetworkOrProcess(t *testing.T) {
forbidden := []string{"net/http", "net/rpc", "net/smtp", "os/exec", "database/sql", "plugin", "syscall"}
for path, imports := range sdkPackages(t) {
for _, imp := range imports {
std := !strings.Contains(strings.SplitN(imp, "/", 2)[0], ".")
if !std && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-sdk/go/") {
t.Errorf("%s imports %s, which is neither the standard library nor mesh-sdk", path, imp)
}
for _, f := range forbidden {
if imp == f {
t.Errorf("%s imports %s", path, imp)
}
}
}
}
}
// 1621/AC2: run with an empty network namespace, the command still writes its verdict: it needs no network. The test
// runs its own binary again under `unshare --net` as an unprivileged user's namespace; where the kernel or the sandbox
// allows no new namespace (a build module's run step is already one with no network), it says so and skips.
func TestTheCommandRunsWithNoNetwork(t *testing.T) {
if os.Getenv("MESH_COMPOSE_HELPER") == "1" {
os.Exit(run(nil, os.Stdin, os.Stdout, os.Stderr))
}
unshare, err := exec.LookPath("unshare")
if err != nil {
t.Skip("no unshare on this machine, so no empty network namespace can be made here")
}
if out, err := exec.Command(unshare, "--net", "--map-root-user", "true").CombinedOutput(); err != nil {
t.Skipf("this machine makes no unprivileged network namespace (%v: %s); a run step's own has no network", err, out)
}
r := request(t, composer.DepthResolve)
cmd := exec.Command(unshare, "--net", "--map-root-user", os.Args[0], "-test.run=^TestTheCommandRunsWithNoNetwork$")
cmd.Env = []string{"MESH_COMPOSE_HELPER=1"}
cmd.Stdin = bytes.NewReader(encode(t, r))
var out, errOut bytes.Buffer
cmd.Stdout, cmd.Stderr = &out, &errOut
if err := cmd.Run(); err != nil {
t.Fatalf("the command in an empty network namespace: %v: %s", err, errOut.String())
}
var v composer.Verdict
if err := json.Unmarshal(out.Bytes(), &v); err != nil || len(v.Cases) != 1 || len(v.Cases[0].Machines) != 3 {
t.Fatalf("no verdict from the command with no network: %v\n%s", err, out.String())
}
names := make([]string, 0, len(v.Cases[0].Machines))
for n := range v.Cases[0].Machines {
names = append(names, n)
}
sort.Strings(names)
t.Logf("composed with no network: %v", names)
}
// -catalogue judges a fallback to a predefined key against the predefined.json at the checkout's root (ADR 0348 §7):
// xterm (mesh-catalog at 835bf22b) falls back to "font-size", accepted where the root lists it and refused where the
// root's list does not.
func TestACatalogueCheckoutJudgesFallbacksAgainstItsPredefinedKeys(t *testing.T) {
raw, err := os.ReadFile("../../catalogue/testdata/catalogue-835bf22b-xterm.module.json")
if err != nil {
t.Fatal(err)
}
for _, c := range []struct {
list string
want bool
}{
{`{"font-size": {"type": "number", "description": "the desktop's font size"}}`, true},
{`{"font-family": {"type": "string", "description": "the desktop's font"}}`, false},
} {
dir := t.TempDir()
if err := os.MkdirAll(filepath.Join(dir, "modules", "xterm"), 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "modules", "xterm", "module.json"), raw, 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, catalogue.PredefinedFile), []byte(c.list), 0o600); err != nil {
t.Fatal(err)
}
code, v, stderr := runJSON(t, []string{"-catalogue", dir}, nil)
if code != 0 {
t.Fatalf("exit %d: %s", code, stderr)
}
if got := v.Cases[0].Manifests["xterm"]; got.Accepted != c.want {
t.Errorf("predefined.json %s: xterm %+v, want accepted %v", c.list, got, c.want)
} }
} }
} }
+1
View File
@@ -0,0 +1 @@
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
+209
View File
@@ -0,0 +1,209 @@
// mesh-hand-compose composes one module for one node by hand, from the inputs the controller keeps, and signs it with the
// operator's key, for the hand path of novox/hq ADR 0539 clause 10: a fingerprint the controller never composed (a
// hand build), while the controller cannot compose its own fix. It composes with mesh-sdk's composition, the one the
// controller writes with (compose.Compositions), and prints the composition, its key and its digest; only with -write
// does it write them, with the operator's credential: the composition into the node's bucket composed-<node>, each
// file over 64 KiB into the node's object bucket, and the digest into mesh-controller_composed. The want is the
// operator's next act, naming that digest; each act is recorded in the hand-act log.
//
// mesh-hand-compose -input <inputs.json> -node <node> -module <module> -fp <src2:...> -key <operator key>
// [-write -nats <url> -creds <operator credential>]
//
// The inputs are a compose.Input as JSON. The key is the operator's ed25519 private key, PKCS#8 in PEM; the
// node-engine trusts it beside the controller's (ADR 0539 clause 3).
package main
import (
"context"
"crypto/ed25519"
"crypto/x509"
"encoding/json"
"encoding/pem"
"errors"
"flag"
"fmt"
"io"
"os"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/compose"
"git.novox.be/novox/mesh-sdk/go/declaration"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
func main() {
if err := run(os.Args[1:], os.Stdout); err != nil {
fmt.Fprintln(os.Stderr, "mesh-hand-compose:", err)
os.Exit(1)
}
}
// printed is what mesh-hand-compose prints: where the composition goes, its digest, the composition, and whether it was
// written.
type printed struct {
Bucket string `json:"bucket"`
Key string `json:"key"`
Digest string `json:"digest"`
Composition json.RawMessage `json:"composition"`
Objects []string `json:"objects,omitempty"`
Written bool `json:"written"`
Next string `json:"next"`
}
func run(args []string, stdout io.Writer) error {
fs := flag.NewFlagSet("mesh-hand-compose", flag.ContinueOnError)
input := fs.String("input", "", "the controller's kept inputs, a compose.Input as JSON")
node := fs.String("node", "", "the node to compose for")
module := fs.String("module", "", "the module to compose")
fp := fs.String("fp", "", "the module's fingerprint, src2: and 64 hex digits")
keyFile := fs.String("key", "", "the operator's ed25519 private key, PKCS#8 in PEM")
write := fs.Bool("write", false, "write the composition, its files and its digest, with the operator's credential")
url := fs.String("nats", nats.DefaultURL, "with -write: the bus")
creds := fs.String("creds", "", "with -write: the operator's credential file")
if err := fs.Parse(args); err != nil {
return err
}
switch {
case *input == "" || *node == "" || *module == "" || *fp == "":
return errors.New("-input, -node, -module and -fp are each required")
case *keyFile == "":
return errors.New("-key is required: a composition made by hand is signed by the operator's key, and the node-engine applies no other")
case *write && *creds == "":
return errors.New("-write needs -creds: the operator's credential is the only hand writer of a composition")
}
key, err := readKey(*keyFile)
if err != nil {
return err
}
key2, err := delivery.ComposedKey(*node, *module, *fp)
if err != nil {
return err
}
bucket, err := delivery.ComposedBucketFor(*node)
if err != nil {
return err
}
raw, err := os.ReadFile(*input)
if err != nil {
return err
}
var in compose.Input
if err := json.Unmarshal(raw, &in); err != nil {
return fmt.Errorf("the inputs in %s are not a compose.Input: %w", *input, err)
}
all, _, err := compose.Compositions(in, *node, map[string]string{*module: *fp}, compose.Reading)
if err != nil {
return err
}
c, ok := all[key2]
if !ok {
return fmt.Errorf("%s does not run %s: its composition holds none of it", *node, *module)
}
c, objects, err := c.ByDigest()
if err != nil {
return err
}
if c, err = delivery.SignComposed(c, key); err != nil {
return err
}
encoded, err := delivery.EncodeComposed(c)
if err != nil {
return err
}
digest, err := delivery.ComposedDigestOf(c)
if err != nil {
return err
}
out := printed{Bucket: bucket, Key: key2, Digest: digest, Composition: encoded,
Next: fmt.Sprintf("write want.%s.%s naming fp %s and composed %s with the operator's credential, and record each act in the hand-act log", *node, *module, *fp, digest)}
for d := range objects {
out.Objects = append(out.Objects, d)
}
if *write {
if err := writeIt(*url, *creds, *node, bucket, key2, encoded, digest, objects); err != nil {
return err
}
out.Written = true
}
enc := json.NewEncoder(stdout)
enc.SetIndent("", " ")
return enc.Encode(out)
}
func readKey(path string) (ed25519.PrivateKey, error) {
raw, err := os.ReadFile(path)
if err != nil {
return nil, err
}
block, _ := pem.Decode(raw)
if block == nil {
return nil, fmt.Errorf("%s holds no PEM block", path)
}
parsed, err := x509.ParsePKCS8PrivateKey(block.Bytes)
if err != nil {
return nil, fmt.Errorf("%s: %w", path, err)
}
key, ok := parsed.(ed25519.PrivateKey)
if !ok {
return nil, fmt.Errorf("%s is not an ed25519 key", path)
}
return key, nil
}
// writeIt writes the files by digest, then the composition, then its digest: a reader that finds the digest finds the
// composition and its files already there.
func writeIt(url, creds, node, bucket, key string, encoded []byte, digest string, objects map[string][]byte) error {
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()
nc, err := nats.Connect(url, nats.UserCredentials(creds), nats.Name("mesh-hand-compose"))
if err != nil {
return err
}
defer nc.Close()
js, err := jetstream.New(nc)
if err != nil {
return err
}
if len(objects) > 0 {
filesBucket, err := declaration.FileBucketFor(node)
if err != nil {
return err
}
store, err := js.ObjectStore(ctx, filesBucket)
if err != nil {
return fmt.Errorf("the node's object bucket %s: %w", filesBucket, err)
}
for d, body := range objects {
name, err := declaration.ObjectName(d)
if err != nil {
return err
}
if _, err := store.PutBytes(ctx, name, body); err != nil {
return fmt.Errorf("the file %s: %w", d, err)
}
}
}
composed, err := js.KeyValue(ctx, bucket)
if err != nil {
return fmt.Errorf("the bucket %s: %w", bucket, err)
}
if _, err := composed.Put(ctx, key, encoded); err != nil {
return fmt.Errorf("%s in %s: %w", key, bucket, err)
}
value, err := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digest, Judgement: delivery.JudgedCannot,
Words: "composed by hand with mesh-hand-compose (ADR 0539 clause 10): the operator writes the want"})
if err != nil {
return err
}
digests, err := js.KeyValue(ctx, delivery.ControllerComposedBucket)
if err != nil {
return fmt.Errorf("the bucket %s: %w", delivery.ControllerComposedBucket, err)
}
if _, err := digests.Put(ctx, key, value); err != nil {
return fmt.Errorf("%s in %s: %w", key, delivery.ControllerComposedBucket, err)
}
return nil
}
+145
View File
@@ -0,0 +1,145 @@
package main
import (
"bytes"
"crypto/ed25519"
"crypto/rand"
"crypto/x509"
"encoding/base64"
"encoding/json"
"encoding/pem"
"os"
"path/filepath"
"sort"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/compose"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/overlay"
)
// aMesh is the Input of mesh-controller's aMesh, as mesh-sdk's compose tests make it (compose/fixture_test.go): two
// machines on the private network, the module the mesh provides for itself assigned to both. The manifest is the one
// the controller provides (overlay.Manifest), read back as the catalogue reads it.
func aMesh(t *testing.T) compose.Input {
t.Helper()
var m catalogue.Manifest
b, _ := json.Marshal(overlay.Manifest())
if err := json.Unmarshal(b, &m); err != nil {
t.Fatal(err)
}
in := compose.Input{Catalogue: map[string]catalogue.Manifest{overlay.Name: m}}
for i, name := range []string{"anchor", "laptop"} {
pub, _, _ := ed25519.GenerateKey(rand.Reader)
in.Nodes = append(in.Nodes, compose.Node{Name: name,
Place: compose.Place{Key: base64.StdEncoding.EncodeToString(pub), Endpoint: name + ".example:51820", Site: "here", Hub: name == "anchor", Address: []string{"10.77.0.1", "10.77.0.2"}[i]},
Assigned: []string{overlay.Name}, Capabilities: map[string]bool{"container-runtime": true, "wireguard": true, "systemd": true}})
}
return in
}
func writeKey(t *testing.T, dir string) ed25519.PublicKey {
t.Helper()
pub, priv, _ := ed25519.GenerateKey(rand.Reader)
der, err := x509.MarshalPKCS8PrivateKey(priv)
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "operator.key"), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der}), 0o600); err != nil {
t.Fatal(err)
}
return pub
}
// 1943/AC5: mesh-hand-compose, given the kept inputs, a node, a module and its fingerprint, prints that module's
// composition for that node signed by the operator's key and its digest: its resources are the controller's for that
// module on that node (compose.Compose), and it verifies against the operator's key. Without --write it writes nothing.
func TestMeshComposeGivesTheControllersResourcesSignedByTheOperator(t *testing.T) {
dir := t.TempDir()
in := aMesh(t)
raw, err := json.Marshal(in)
if err != nil {
t.Fatal(err)
}
input := filepath.Join(dir, "input.json")
if err := os.WriteFile(input, raw, 0o600); err != nil {
t.Fatal(err)
}
operator := writeKey(t, dir)
fp := delivery.SourcePrefix2 + strings.Repeat("a", 64)
var out bytes.Buffer
if err := run([]string{"-input", input, "-node", "laptop", "-module", overlay.Name, "-fp", fp,
"-key", filepath.Join(dir, "operator.key")}, &out); err != nil {
t.Fatal(err)
}
var printed struct {
Key string `json:"key"`
Bucket string `json:"bucket"`
Digest string `json:"digest"`
Composition json.RawMessage `json:"composition"`
Written bool `json:"written"`
}
if err := json.Unmarshal(out.Bytes(), &printed); err != nil {
t.Fatalf("%s: %v", out.String(), err)
}
c, err := delivery.DecodeComposed(printed.Composition)
if err != nil {
t.Fatal(err)
}
if digest, err := delivery.VerifyComposed(c, []ed25519.PublicKey{operator}); err != nil || digest != printed.Digest {
t.Errorf("the composition verifies %q, %v against the operator's key; printed %s", digest, err, printed.Digest)
}
wantKey, _ := delivery.ComposedKey("laptop", overlay.Name, fp)
if printed.Key != wantKey || printed.Bucket != "composed-laptop" || printed.Written {
t.Errorf("printed key %s in %s, written %v", printed.Key, printed.Bucket, printed.Written)
}
d, _, err := compose.Compose(in, "laptop")
if err != nil {
t.Fatal(err)
}
var want, got []string
for _, r := range d.Resources {
want = append(want, r["id"].(string))
}
for _, r := range c.Resources {
var head struct{ ID string }
_ = json.Unmarshal(r, &head)
got = append(got, head.ID)
}
sort.Strings(want)
sort.Strings(got)
if len(got) == 0 || strings.Join(got, " ") != strings.Join(want, " ") {
t.Errorf("mesh-hand-compose gives %v, the controller's composition %v", got, want)
}
}
// 1943/AC5: mesh-hand-compose refuses what it cannot compose, naming why: no key, a fingerprint not of the new form, a
// module the node does not run, a node the mesh does not have.
func TestMeshComposeRefuses(t *testing.T) {
dir := t.TempDir()
raw, _ := json.Marshal(aMesh(t))
input := filepath.Join(dir, "input.json")
_ = os.WriteFile(input, raw, 0o600)
writeKey(t, dir)
key := filepath.Join(dir, "operator.key")
fp := delivery.SourcePrefix2 + strings.Repeat("a", 64)
for _, c := range []struct {
args []string
why string
}{
{[]string{"-input", input, "-node", "laptop", "-module", overlay.Name, "-fp", fp}, "-key"},
{[]string{"-input", input, "-node", "laptop", "-module", overlay.Name, "-fp", "src2:1", "-key", key}, "fingerprint"},
{[]string{"-input", input, "-node", "laptop", "-module", "mesh-mail", "-fp", fp, "-key", key}, "does not run mesh-mail"},
{[]string{"-input", input, "-node", "nowhere", "-module", overlay.Name, "-fp", fp, "-key", key}, "nowhere"},
{[]string{"-input", input, "-node", "laptop", "-module", overlay.Name, "-fp", fp, "-key", key, "-write"}, "-creds"},
} {
var out bytes.Buffer
if err := run(c.args, &out); err == nil || !strings.Contains(err.Error(), c.why) {
t.Errorf("mesh-hand-compose %v = %v, want a refusal saying %q", c.args, err, c.why)
}
}
}
+1 -1
View File
@@ -13,7 +13,7 @@ import (
// the controller composes it, split into each module's composition at the fingerprint fps gives it and the node's own // the controller composes it, split into each module's composition at the fingerprint fps gives it and the node's own
// (`<node>.node`), by key (delivery.ComposedKey). Each resource says its module, whatever the node reported it reads. // (`<node>.node`), by key (delivery.ComposedKey). Each resource says its module, whatever the node reported it reads.
// A module fps gives no fingerprint for is not composed and is named in skipped, so a caller composing one module // A module fps gives no fingerprint for is not composed and is named in skipped, so a caller composing one module
// (mesh-compose) gets that one. Each module's Data says whether its manifest declares data. The compositions are not // (mesh-hand-compose) gets that one. Each module's Data says whether its manifest declares data. The compositions are not
// signed and not yet by digest: the writer does both (delivery.SignComposed, Composed.ByDigest). // signed and not yet by digest: the writer does both (delivery.SignComposed, Composed.ByDigest).
func Compositions(in Input, node string, fps map[string]string, mode Mode) (out map[string]delivery.Composed, skipped []string, err error) { func Compositions(in Input, node string, fps map[string]string, mode Mode) (out map[string]delivery.Composed, skipped []string, err error) {
reading := in reading := in
+347
View File
@@ -0,0 +1,347 @@
// Package composer is mesh-sdk's composition and validator as one function over a plain request (novox/hq ADR 0482
// clause 4, issue 1621): what the command mesh-compose runs, so a program that links any one mesh-sdk can have the
// composition of another commit judged by building that commit's command and reading its verdict.
//
// **It reads its request and nothing else.** No bus, store, network, file or environment: the manifests come inside the
// request (raw, as their repository holds them), the rest of the mesh is a compose.Input, and the answer is a value.
// A manifest named only by its digest in the delivery indexer's bucket is filled before the run by Fill, by whoever
// holds the bus (the Go build module's compose job); Run refuses a digest left unfilled.
package composer
import (
"encoding/json"
"errors"
"fmt"
"slices"
"sort"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/compose"
"git.novox.be/novox/mesh-sdk/go/sdkversion"
)
// Version is the version of a Request and of a Verdict; a reader refuses another.
const Version = 1
// Package is the import path of the command, which a compose job builds at a commit (in mesh-sdk's own module, or in
// a module that requires mesh-sdk, at the version it pins).
const Package = "git.novox.be/novox/mesh-sdk/go/cmd/mesh-compose"
// The depths of a composition.
const (
// DepthResolve resolves each machine's set over the rest of the mesh (compose.Resolve): what the assigner judges
// while the facts snapshot is not on the bus (issue 1461).
DepthResolve = "resolve"
// DepthCompose composes each machine's whole declaration and runs the node-engine's validator over it
// (compose.Compose).
DepthCompose = "compose"
)
// Request is everything one run composes: the rest of the mesh, one or more cases of manifests composed over it, and
// the machines composed in each case.
type Request struct {
V int `json:"v"`
// Input is the composition's input without its catalogue (and without each node's kept manifests): the cases give
// those. A Catalogue or Kept it holds is ignored.
Input compose.Input `json:"input"`
// Cases are composed one after the other over Input, each with its own manifests; their names are unique.
Cases []Case `json:"cases"`
// Machines are the nodes composed in every case; none means the manifests are only read.
Machines []string `json:"machines,omitempty"`
// Depth is DepthResolve or DepthCompose; empty is DepthCompose.
Depth string `json:"depth,omitempty"`
// HoldingsFromClaims adds to Input's holdings every seat an assigned module claims, held on its node (the
// assigner's input, which has no holdings of record).
HoldingsFromClaims bool `json:"holdings_from_claims,omitempty"`
// Declarations asks each composed machine's declared paths (DepthCompose only).
Declarations bool `json:"declarations,omitempty"`
// Predefined is the list of predefined keys at the catalogue's root (predefined.json, novox/hq ADR 0348 §7) that
// every config value falling back to a key is judged against, when the asker has it (mesh-compose -catalogue
// reads it). Without it a fallback is read as catalogue.ParseManifestForInputs reads it: against the list the mesh
// published when this process holds one, else each key as its value declares it, for the merge gate judges the
// fallback in the catalogue and a manifest that only carries one is not refused here.
Predefined catalogue.PredefinedList `json:"predefined,omitempty"`
}
// Case is one catalogue composed: every module's manifest by name, and per node the manifests kept at the build it runs.
type Case struct {
Name string `json:"name"`
Manifests map[string]Manifest `json:"manifests"`
Kept map[string]map[string]Manifest `json:"kept,omitempty"`
}
// Manifest is a module's manifest as its repository holds it (Raw, the bytes), or the digest the delivery indexer keeps
// it under (Digest), which Fill replaces by the bytes. On the wire it is the manifest itself, or {"digest": "..."}.
type Manifest struct {
Raw json.RawMessage
Digest string
}
// MarshalJSON writes the raw manifest, or {"digest": ...} when only the digest is known.
func (m Manifest) MarshalJSON() ([]byte, error) {
if len(m.Raw) > 0 {
return m.Raw, nil
}
return json.Marshal(map[string]string{"digest": m.Digest})
}
// UnmarshalJSON reads {"digest": "..."} (an object with that one key) as a digest, and any other value as the raw
// manifest.
func (m *Manifest) UnmarshalJSON(raw []byte) error {
var probe map[string]json.RawMessage
if json.Unmarshal(raw, &probe) == nil && len(probe) == 1 {
if d, ok := probe["digest"]; ok {
var digest string
if err := json.Unmarshal(d, &digest); err != nil {
return fmt.Errorf("a manifest's digest is not a string: %w", err)
}
*m = Manifest{Digest: digest}
return nil
}
}
*m = Manifest{Raw: slices.Clone(raw)}
return nil
}
// Verdict is what one run answers: the mesh-sdk the command was built with, and each case's verdict in the order asked.
type Verdict struct {
V int `json:"v"`
SDK string `json:"sdk"`
Cases []CaseVerdict `json:"cases"`
}
// CaseVerdict is one case's: every manifest read or refused, every machine composed or not.
type CaseVerdict struct {
Name string `json:"name"`
Manifests map[string]ManifestRead `json:"manifests"`
Machines map[string]Machine `json:"machines,omitempty"`
}
// ManifestRead is one manifest as this mesh-sdk reads it: accepted, or refused with the reader's words.
type ManifestRead struct {
Accepted bool `json:"accepted"`
Refused string `json:"refused,omitempty"`
}
// Machine is one machine composed: whether it composes (resolves, at DepthResolve), why not, the node-engine
// validator's problems and the composition's refusals (DepthCompose), and its declared paths when asked.
type Machine struct {
Composes bool `json:"composes"`
Reason string `json:"reason,omitempty"`
Problems []string `json:"problems,omitempty"`
Refusals []string `json:"refusals,omitempty"`
Declared []Path `json:"declared,omitempty"`
}
// Path is one path a declaration puts on the machine, with its kind (the resource's type), owner, group and mode as
// declared, and the module it is of.
type Path struct {
Path string `json:"path"`
Kind string `json:"kind"`
Owner string `json:"owner,omitempty"`
Group string `json:"group,omitempty"`
Mode string `json:"mode,omitempty"`
Module string `json:"module,omitempty"`
}
// ErrUnfilled is a manifest named by a digest that nobody replaced by its bytes.
var ErrUnfilled = errors.New("composer: a manifest is named by its digest and was not filled")
// Decode reads a request; it refuses one that is not JSON or of another version.
func Decode(data []byte) (Request, error) {
var r Request
if err := json.Unmarshal(data, &r); err != nil {
return Request{}, fmt.Errorf("composer: the request is not JSON of its shape: %w", err)
}
if r.V != Version {
return Request{}, fmt.Errorf("composer: a request of version %d, this composer reads %d", r.V, Version)
}
return r, nil
}
// Fill replaces every manifest named by its digest by the bytes get answers for that digest; a digest get cannot answer
// stops it, named.
func Fill(r *Request, get func(digest string) ([]byte, error)) error {
fill := func(m map[string]Manifest) error {
for name, man := range m {
if len(man.Raw) > 0 {
continue
}
raw, err := get(man.Digest)
if err != nil {
return fmt.Errorf("composer: the manifest of %s (digest %s): %w", name, man.Digest, err)
}
m[name] = Manifest{Raw: raw}
}
return nil
}
for _, c := range r.Cases {
if err := fill(c.Manifests); err != nil {
return err
}
for _, kept := range c.Kept {
if err := fill(kept); err != nil {
return err
}
}
}
return nil
}
// Run composes every case of r and answers the verdict, its SDK this binary's mesh-sdk. An error is a request it
// cannot run (an unknown depth, a case named twice, a manifest left as a digest); a machine that does not compose is a
// verdict.
func Run(r Request) (Verdict, error) {
depth := r.Depth
if depth == "" {
depth = DepthCompose
}
if depth != DepthResolve && depth != DepthCompose {
return Verdict{}, fmt.Errorf("composer: the depth %q is not resolve or compose", r.Depth)
}
v := Verdict{V: Version, SDK: sdkversion.Version()}
seen := map[string]bool{}
for _, c := range r.Cases {
if seen[c.Name] {
return Verdict{}, fmt.Errorf("composer: the case %q is named twice", c.Name)
}
seen[c.Name] = true
cv, err := runCase(r, c, depth)
if err != nil {
return Verdict{}, err
}
v.Cases = append(v.Cases, cv)
}
return v, nil
}
// readManifest reads one manifest as this run reads every one: against the request's list of predefined keys when it
// has one, else as the indexer and the builder read it (Request.Predefined).
func readManifest(raw []byte, predefined catalogue.PredefinedList) (catalogue.Manifest, error) {
if predefined != nil {
return catalogue.ParseManifestWith(raw, predefined)
}
return catalogue.ParseManifestForInputs(raw)
}
func parse(m map[string]Manifest, reads map[string]ManifestRead, predefined catalogue.PredefinedList) (map[string]catalogue.Manifest, error) {
out := map[string]catalogue.Manifest{}
for name, man := range m {
if len(man.Raw) == 0 {
return nil, fmt.Errorf("%w: %s (digest %s)", ErrUnfilled, name, man.Digest)
}
parsed, err := readManifest(man.Raw, predefined)
if reads != nil {
if err != nil {
reads[name] = ManifestRead{Refused: err.Error()}
} else {
reads[name] = ManifestRead{Accepted: true}
}
}
if err == nil {
out[name] = parsed
}
}
return out, nil
}
func runCase(r Request, c Case, depth string) (CaseVerdict, error) {
cv := CaseVerdict{Name: c.Name, Manifests: map[string]ManifestRead{}}
shelf, err := parse(c.Manifests, cv.Manifests, r.Predefined)
if err != nil {
return CaseVerdict{}, err
}
in := r.Input
in.Catalogue = shelf
in.Nodes = slices.Clone(in.Nodes)
in.Holdings = slices.Clone(in.Holdings)
for i := range in.Nodes {
in.Nodes[i].Kept = nil
if kept := c.Kept[in.Nodes[i].Name]; len(kept) > 0 {
k, err := parse(kept, nil, r.Predefined)
if err != nil {
return CaseVerdict{}, err
}
in.Nodes[i].Kept = k
}
}
if r.HoldingsFromClaims {
in.Holdings = append(in.Holdings, HoldingsOf(shelf, in.Nodes)...)
}
if len(r.Machines) == 0 {
return cv, nil
}
cv.Machines = map[string]Machine{}
for _, node := range r.Machines {
cv.Machines[node] = machine(in, node, depth, r.Declarations)
}
return cv, nil
}
// HoldingsOf is every seat an assigned module claims, held on the node it is assigned to, in node order: the holdings
// of an input that has none of record (the assigner's, mesh-delivery-assigner internal/assign input).
func HoldingsOf(shelf map[string]catalogue.Manifest, nodes []compose.Node) []catalogue.Held {
var out []catalogue.Held
sorted := slices.Clone(nodes)
sort.Slice(sorted, func(i, j int) bool { return sorted[i].Name < sorted[j].Name })
for _, n := range sorted {
for _, mod := range n.Assigned {
for _, cl := range shelf[mod].Claims {
scope := cl.Scope
if seat, ok := catalogue.SeatNamed(cl.Name); ok && scope == "" {
scope = seat.Scope
}
out = append(out, catalogue.Held{Claim: cl.Name, Scope: scope, Node: n.Name, Module: mod})
}
}
}
return out
}
func machine(in compose.Input, node, depth string, declarations bool) Machine {
if depth == DepthResolve {
if _, err := compose.Resolve(in, node); err != nil {
return Machine{Reason: err.Error()}
}
return Machine{Composes: true}
}
d, problems, err := compose.Compose(in, node)
if err != nil {
return Machine{Reason: err.Error()}
}
m := Machine{Composes: len(problems) == 0, Problems: problems, Refusals: d.Refusals()}
if len(problems) > 0 {
m.Reason = "the node-engine's validator refuses the declaration"
}
if declarations {
m.Declared = pathsOf(d)
}
return m
}
// pathsOf is every resource of a declaration that names a path, with its owner, group and mode as declared, sorted by
// path.
func pathsOf(d compose.Declared) []Path {
var out []Path
str := func(r map[string]any, key string) string {
if v, ok := r[key]; ok && v != nil {
return fmt.Sprint(v)
}
return ""
}
for _, r := range d.Resources {
p := str(r, "path")
if p == "" {
continue
}
out = append(out, Path{Path: p, Kind: str(r, "type"), Owner: str(r, "owner"), Group: str(r, "group"),
Mode: str(r, "mode"), Module: d.ModuleOf[str(r, "id")]})
}
sort.Slice(out, func(i, j int) bool {
if out[i].Path != out[j].Path {
return out[i].Path < out[j].Path
}
return out[i].Kind < out[j].Kind
})
return out
}
+137
View File
@@ -0,0 +1,137 @@
package composer
import (
"encoding/json"
"errors"
"os"
"reflect"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/compose"
)
const demo = `{"module": "demo", "version": "1", "claims": [{"name": "node-files"}]}`
// A manifest named by its digest goes on the wire as {"digest": ...}, and Fill puts the bytes the indexer keeps in its
// place; a raw manifest goes as itself.
func TestAManifestIsItsBytesOrItsDigest(t *testing.T) {
r := Request{V: Version, Cases: []Case{{Name: "change", Manifests: map[string]Manifest{
"demo": {Digest: "d1"}, "raw": {Raw: json.RawMessage(`{"module":"raw","version":"1"}`)}},
Kept: map[string]map[string]Manifest{"laptop": {"demo": {Digest: "d1"}}}}}}
wire, err := json.Marshal(r)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(wire), `"demo":{"digest":"d1"}`) || !strings.Contains(string(wire), `"raw":{"module":"raw","version":"1"}`) {
t.Fatalf("on the wire: %s", wire)
}
back, err := Decode(wire)
if err != nil {
t.Fatal(err)
}
if _, err := Run(back); !errors.Is(err, ErrUnfilled) {
t.Fatalf("a digest left unfilled runs: %v", err)
}
asked := 0
if err := Fill(&back, func(d string) ([]byte, error) {
asked++
if d != "d1" {
return nil, errors.New("no such digest")
}
return []byte(demo), nil
}); err != nil {
t.Fatal(err)
}
if asked != 2 || string(back.Cases[0].Manifests["demo"].Raw) != demo || string(back.Cases[0].Kept["laptop"]["demo"].Raw) != demo {
t.Fatalf("filled %d times: %+v", asked, back.Cases[0])
}
bad := Request{V: Version, Cases: []Case{{Name: "c", Manifests: map[string]Manifest{"x": {Digest: "nope"}}}}}
if err := Fill(&bad, func(string) ([]byte, error) { return nil, errors.New("not there") }); err == nil || !strings.Contains(err.Error(), "nope") {
t.Fatalf("a digest the indexer does not hold: %v", err)
}
}
// The input travels as JSON and comes back equal, so the command composes what its caller meant.
func TestTheInputTravelsAsJSON(t *testing.T) {
in := compose.Input{
Nodes: []compose.Node{{Name: "anchor", Assigned: []string{"nats"}, Capabilities: map[string]bool{"systemd": true},
Place: compose.Place{Endpoint: "anchor.example:51820", Site: "here", Hub: true, Address: "10.77.0.1"},
Pins: map[string]catalogue.Chosen{"bus": {Node: "anchor", Module: "nats"}},
Settings: map[string][]catalogue.Layer{"nats": {{Values: map[string]any{"port": float64(4222)}}}}}},
Holdings: []catalogue.Held{{Claim: "mesh-broker", Scope: "mesh", Node: "anchor", Module: "nats"}},
Issued: []compose.Issued{{Provision: "bus", Consumer: "laptop", ConsumerModule: "x", Provider: "anchor"}},
}
raw, err := json.Marshal(Request{V: Version, Input: in})
if err != nil {
t.Fatal(err)
}
back, err := Decode(raw)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(back.Input, in) {
t.Fatalf("the input came back as\n%+v\nwant\n%+v", back.Input, in)
}
}
// The seats a node's assigned modules claim are held there when the request says the holdings come from the claims.
func TestHoldingsComeFromTheClaims(t *testing.T) {
m, err := catalogue.ParseManifest([]byte(demo))
if err != nil {
t.Fatal(err)
}
got := HoldingsOf(map[string]catalogue.Manifest{"demo": m}, []compose.Node{{Name: "b", Assigned: []string{"demo"}}, {Name: "a", Assigned: []string{"demo"}}})
if len(got) != 2 || got[0].Node != "a" || got[0].Claim != "node-files" || got[0].Scope != "node" {
t.Fatalf("holdings = %+v", got)
}
}
// A request the command cannot run is refused, never answered as a verdict: another version, a case named twice, an
// unknown depth.
func TestARequestItCannotRunIsRefused(t *testing.T) {
if _, err := Decode([]byte(`{"v": 2}`)); err == nil {
t.Error("a request of version 2 is read")
}
if _, err := Run(Request{V: Version, Cases: []Case{{Name: "a"}, {Name: "a"}}}); err == nil {
t.Error("a case named twice runs")
}
if _, err := Run(Request{V: Version, Depth: "deep"}); err == nil {
t.Error("an unknown depth runs")
}
v, err := Run(Request{V: Version, Cases: []Case{{Name: "only-read", Manifests: map[string]Manifest{"demo": {Raw: json.RawMessage(demo)}}}}})
if err != nil || len(v.Cases) != 1 || !v.Cases[0].Manifests["demo"].Accepted || v.Cases[0].Machines != nil || v.SDK == "" {
t.Fatalf("a request with no machines: %+v, %v", v, err)
}
}
// A manifest whose config value falls back to a predefined key (novox/hq ADR 0348 §7) is read, not refused for the
// list of predefined keys at the catalogue's root that a run is never handed with its manifests: the merge gate judges
// the fallback in the catalogue. xterm's module.json from mesh-catalog at 835bf22b falls back to "font-size"; a group
// touching xterm or pacman failed its composition on this (group check 18df81380db3). A list in the request is the one
// judged against, so a fallback it does not name is still refused.
func TestAManifestFallingBackToAPredefinedKeyIsRead(t *testing.T) {
raw, err := os.ReadFile("../catalogue/testdata/catalogue-835bf22b-xterm.module.json")
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(raw), `"predefined"`) {
t.Fatal("the fixture names no predefined key")
}
r := Request{V: Version, Cases: []Case{{Name: "change", Manifests: map[string]Manifest{"xterm": {Raw: raw}}}}}
v, err := Run(r)
if err != nil {
t.Fatal(err)
}
if got := v.Cases[0].Manifests["xterm"]; !got.Accepted {
t.Fatalf("xterm: %+v, want accepted", got)
}
r.Predefined = catalogue.PredefinedList{"something-else": {Type: "string", Description: "x"}}
if v, err = Run(r); err != nil {
t.Fatal(err)
}
if got := v.Cases[0].Manifests["xterm"]; got.Accepted || !strings.Contains(got.Refused, "not on the list of predefined keys") {
t.Fatalf("xterm judged against a list without its key: %+v, want refused naming the list", got)
}
}
+2 -2
View File
@@ -14,7 +14,7 @@ import (
// resolver, identities and groups) under `<node>.node`. Beside it, under the same key in `mesh-controller_composed`, // resolver, identities and groups) under `<node>.node`. Beside it, under the same key in `mesh-controller_composed`,
// which the assigner reads, the controller writes the composition's digest and its judgement of the move from the // which the assigner reads, the controller writes the composition's digest and its judgement of the move from the
// composition the node runs now (ComposedDigest). The controller writes both by compare-and-set while it holds the // composition the node runs now (ComposedDigest). The controller writes both by compare-and-set while it holds the
// lease; the operator's credential is a second writer on the hand path alone (clause 10, mesh-compose). // lease; the operator's credential is a second writer on the hand path alone (clause 10, mesh-hand-compose).
// //
// A composition is derived state (clause 4): its key and its digest are deleted by the controller when its fingerprint // A composition is derived state (clause 4): its key and its digest are deleted by the controller when its fingerprint
// is no longer among the module's five newest and no want names it, and each bucket keeps five revisions of a key. // is no longer among the module's five newest and no want names it, and each bucket keeps five revisions of a key.
@@ -143,7 +143,7 @@ func checkDigest(digest, of string) error {
// Policy is the module's upgrade policy as the controller holds it (ADR 0236, ADR 0539 clause 14), with PolicyFrom // Policy is the module's upgrade policy as the controller holds it (ADR 0236, ADR 0539 clause 14), with PolicyFrom
// where it comes from and PolicyWhy why: the assigner holds a move of a module of policy record from it, the operator's // where it comes from and PolicyWhy why: the assigner holds a move of a module of policy record from it, the operator's
// own choice (`upgrade` with policy) included. Rewritten when the policy changes. Empty on a composition made by hand // own choice (`upgrade` with policy) included. Rewritten when the policy changes. Empty on a composition made by hand
// (mesh-compose), whose want the operator writes. // (mesh-hand-compose), whose want the operator writes.
type ComposedDigest struct { type ComposedDigest struct {
V int `json:"v"` V int `json:"v"`
Digest string `json:"digest"` Digest string `json:"digest"`
+1 -1
View File
@@ -7,7 +7,7 @@ import (
) )
// A composition's signature (novox/hq ADR 0539 clause 3): the controller signs each composition it writes with its // A composition's signature (novox/hq ADR 0539 clause 3): the controller signs each composition it writes with its
// key, and the operator signs one made by hand with mesh-compose with theirs. The node-engine trusts exactly these two // key, and the operator signs one made by hand with mesh-hand-compose with theirs. The node-engine trusts exactly these two
// signers and applies nothing of a composition whose signature fails or whose signer is neither. The signature is an // signers and applies nothing of a composition whose signature fails or whose signer is neither. The signature is an
// ed25519 signature, as a declaration's is, of the composition's digest (ComposedDigestOf) after a context line, so a // ed25519 signature, as a declaration's is, of the composition's digest (ComposedDigestOf) after a context line, so a
// signature made for anything else never verifies here. // signature made for anything else never verifies here.
+76 -8
View File
@@ -1,8 +1,11 @@
package delivery package delivery
import ( import (
"crypto/sha256"
"encoding/hex"
"encoding/json" "encoding/json"
"fmt" "fmt"
"path"
"regexp" "regexp"
"sort" "sort"
"strings" "strings"
@@ -154,8 +157,8 @@ func checkAskID(id string) error {
} }
// StepAsk is the message on a build seat's stream: who asked what, of which kind and language, and the one whole job. // StepAsk is the message on a build seat's stream: who asked what, of which kind and language, and the one whole job.
// Exactly one of Build and Check is set: a delivery ask is a build; a check ask a build (a candidate's fingerprint) or a // Exactly one of Build, Check and Compose is set: a delivery ask is a build; a check ask a build (a candidate's
// check; a recheck ask a check; an image ask is always a build. // fingerprint), a check or, on the Go seat, a compose; a recheck ask a check; an image ask is always a build.
type StepAsk struct { type StepAsk struct {
V int `json:"v"` V int `json:"v"`
ID string `json:"id"` ID string `json:"id"`
@@ -163,10 +166,48 @@ type StepAsk struct {
Language string `json:"language"` Language string `json:"language"`
// Asker is the module that put the ask (mesh-delivery-builder or mesh-delivery-checker); the stream's grant is what // Asker is the module that put the ask (mesh-delivery-builder or mesh-delivery-checker); the stream's grant is what
// makes it true, this field only names it to a reader. // makes it true, this field only names it to a reader.
Asker string `json:"asker"` Asker string `json:"asker"`
At time.Time `json:"at"` At time.Time `json:"at"`
Build *BuildJob `json:"build,omitempty"` Build *BuildJob `json:"build,omitempty"`
Check *CheckJob `json:"check,omitempty"` Check *CheckJob `json:"check,omitempty"`
Compose *ComposeJob `json:"compose,omitempty"`
}
// ComposeJob is a compose ask's job (novox/hq ADR 0482 clause 4, issue 1622): fetch the commit, build the composer
// command (Package) in the Go module at Dir of the fetched tree, once per ComposerKey, run it in a run step shut off
// with Request on its standard input, and answer its standard output whole (ComposeAnswer). The assigner asks it, to
// compose a group with its mesh-sdk member's head, and to read the group's manifests with the mesh-sdk the running
// controller and node-engine pin (ADR 0482 clause 5, issue 1623).
type ComposeJob struct {
// Fetch is the fetch tool's arguments: the repository and the full commit whose mesh-sdk composes (mesh-sdk itself
// at a head, or a module that requires mesh-sdk, at the version it pins).
Fetch map[string]any `json:"fetch"`
// Dir is the directory of the Go module the composer is built in, relative to the repository's root ("go" for
// mesh-sdk, RootPath for a module at the root); Package is the composer's import path (mesh-sdk composer.Package).
Dir string `json:"dir"`
Package string `json:"package"`
// Request is the composer's request (mesh-sdk composer.Request), its manifests named by their digests in the
// indexer's ManifestsBucket: the build module fills them before the run (composer.Fill).
Request json.RawMessage `json:"request"`
// Reader names, for a reader of the ask, whose composition this is: the mesh-sdk head of a group, or a module whose
// mesh-sdk reads the group's manifests (mesh-controller, mesh-host).
Reader string `json:"reader,omitempty"`
}
// ComposeAnswer is a compose ask's step result: whether this run built the composer or took the one kept under its
// key, the key, and the composer's verdict (mesh-sdk composer.Verdict) whole.
type ComposeAnswer struct {
Built bool `json:"built"`
Key string `json:"key"`
Verdict json.RawMessage `json:"verdict"`
}
// ComposerKey is the key a build module keeps a built composer under: the repository, commit, directory and package it
// was built from and the toolchain it was built with, hashed (`composer-` and 16 hex digits), so each head is built once
// on a node (ADR 0482 clause 4).
func ComposerKey(repository, commit, dir, pkg string, toolchain ToolchainUsed) string {
sum := sha256.Sum256([]byte(strings.Join([]string{repository, commit, dir, pkg, toolchain.Version, toolchain.Recipe}, "\n")))
return "composer-" + hex.EncodeToString(sum[:8])
} }
// BuildJob is a build ask's job: fetch the commit (unless every artifact is a copy), then build or copy each artifact, // BuildJob is a build ask's job: fetch the commit (unless every artifact is a copy), then build or copy each artifact,
@@ -232,9 +273,17 @@ func (a StepAsk) check() error {
if a.At.IsZero() { if a.At.IsZero() {
return fmt.Errorf("delivery: the ask %s names no time", a.ID) return fmt.Errorf("delivery: the ask %s names no time", a.ID)
} }
jobs := 0
for _, set := range []bool{a.Build != nil, a.Check != nil, a.Compose != nil} {
if set {
jobs++
}
}
switch { switch {
case (a.Build == nil) == (a.Check == nil): case jobs != 1:
return fmt.Errorf("delivery: the ask %s holds a build or a check, exactly one", a.ID) return fmt.Errorf("delivery: the ask %s holds a build, a check or a compose, exactly one", a.ID)
case a.Compose != nil && (a.Kind != AskCheck || a.Language != AskGo):
return fmt.Errorf("delivery: the ask %s is a compose, which the Go seat's check stream alone carries", a.ID)
case a.Build != nil && a.Kind == AskRecheck: case a.Build != nil && a.Kind == AskRecheck:
return fmt.Errorf("delivery: the ask %s is a build on the recheck stream, which carries checks alone", a.ID) return fmt.Errorf("delivery: the ask %s is a build on the recheck stream, which carries checks alone", a.ID)
case a.Check != nil && a.Kind == AskDelivery: case a.Check != nil && a.Kind == AskDelivery:
@@ -245,9 +294,28 @@ func (a StepAsk) check() error {
if a.Build != nil { if a.Build != nil {
return a.Build.check(a.ID) return a.Build.check(a.ID)
} }
if a.Compose != nil {
return a.Compose.check(a.ID)
}
return a.Check.check(a.ID) return a.Check.check(a.ID)
} }
func (c ComposeJob) check(id string) error {
if err := checkFetchArgs(id, c.Fetch); err != nil {
return err
}
if c.Dir == "" || c.Dir != path.Clean(c.Dir) || strings.HasPrefix(c.Dir, "/") || c.Dir == ".." || strings.HasPrefix(c.Dir, "../") {
return fmt.Errorf("delivery: the compose ask %s names no clean directory inside the repository (%q)", id, c.Dir)
}
if c.Package == "" {
return fmt.Errorf("delivery: the compose ask %s names no package to build", id)
}
if !json.Valid(c.Request) || len(c.Request) == 0 || c.Request[0] != '{' {
return fmt.Errorf("delivery: the compose ask %s holds no request object", id)
}
return nil
}
func (b BuildJob) check(id string) error { func (b BuildJob) check(id string) error {
if err := checkName("module", b.Module); err != nil { if err := checkName("module", b.Module); err != nil {
return err return err
+76
View File
@@ -0,0 +1,76 @@
package delivery
import (
"encoding/json"
"reflect"
"strings"
"testing"
"time"
)
func composeAsk() StepAsk {
return StepAsk{ID: "mh3k2x9qa1b2c5", Kind: AskCheck, Language: AskGo, Asker: "mesh-delivery-assigner",
At: time.Date(2026, 10, 24, 3, 0, 0, 0, time.UTC),
Compose: &ComposeJob{Fetch: map[string]any{"repository": "novox/mesh-sdk", "commit": askCommit},
Dir: "go", Package: "git.novox.be/novox/mesh-sdk/go/cmd/mesh-compose",
Request: json.RawMessage(`{"v":1,"cases":[{"name":"change","manifests":{"demo":{"digest":"d1"}}}]}`), Reader: "mesh-sdk head"}}
}
// 1622: a compose ask encodes and decodes with every field.
func TestAComposeAskRoundTrips(t *testing.T) {
a := composeAsk()
raw, err := EncodeStepAsk(a)
if err != nil {
t.Fatal(err)
}
got, err := DecodeStepAsk(raw)
if err != nil {
t.Fatal(err)
}
a.V = StepAskVersion
if !reflect.DeepEqual(got, a) {
t.Fatalf("round trip:\n got %+v\nwant %+v", got, a)
}
}
// 1622: a compose ask goes only on the Go seat's check stream, holds no other job beside it, fetches a full commit,
// builds a package in a clean directory of the repository and carries a request object.
func TestAComposeAskIsRefusedWhereItDoesNotBelong(t *testing.T) {
for name, change := range map[string]func(*StepAsk){
"on the delivery stream": func(a *StepAsk) { a.Kind = AskDelivery },
"on the recheck stream": func(a *StepAsk) { a.Kind = AskRecheck },
"on the Node seat": func(a *StepAsk) { a.Language = AskNode },
"beside a check": func(a *StepAsk) { a.Check = checkAsk().Check },
"with a short commit": func(a *StepAsk) { a.Compose.Fetch["commit"] = "0123456" },
"in no directory": func(a *StepAsk) { a.Compose.Dir = "" },
"outside the repository": func(a *StepAsk) { a.Compose.Dir = "../x" },
"with no package": func(a *StepAsk) { a.Compose.Package = "" },
"with no request": func(a *StepAsk) { a.Compose.Request = nil },
"with a request not JSON": func(a *StepAsk) { a.Compose.Request = json.RawMessage(`[1]`) },
} {
a := composeAsk()
change(&a)
if _, err := EncodeStepAsk(a); err == nil {
t.Errorf("a compose ask %s is encoded", name)
}
}
}
// 1622/AC3: a composer is kept under one key per repository, commit, directory, package and toolchain, so two asks of
// one head build it once on a node, and another head or toolchain builds its own.
func TestAComposerIsKeptByWhatItWasBuiltFrom(t *testing.T) {
tc := ToolchainUsed{Version: "go1.26.0", Recipe: "3"}
k := ComposerKey("novox/mesh-sdk", askCommit, "go", "p", tc)
if k != ComposerKey("novox/mesh-sdk", askCommit, "go", "p", tc) || !strings.HasPrefix(k, "composer-") || len(k) != len("composer-")+16 {
t.Fatalf("key = %q", k)
}
for _, other := range []string{
ComposerKey("novox/mesh-sdk", strings.Repeat("1", 40), "go", "p", tc),
ComposerKey("novox/mesh-controller", askCommit, ".", "p", tc),
ComposerKey("novox/mesh-sdk", askCommit, "go", "p", ToolchainUsed{Version: "go1.26.1", Recipe: "3"}),
} {
if other == k {
t.Errorf("two composers built from different inputs share the key %s", k)
}
}
}
+6 -1
View File
@@ -15,7 +15,12 @@
// `mesh/repo-check` runs; a repository missing there fails the test, never skips it. Which clones a // `mesh/repo-check` runs; a repository missing there fails the test, never skips it. Which clones a
// check gets is chosen from the inventory: mesh-catalog by the source of the `nats` module, mesh-host // check gets is chosen from the inventory: mesh-catalog by the source of the `nats` module, mesh-host
// by the source of `mesh-host`, and mesh-sdk as the controller's sibling at the commit the controller's // by the source of `mesh-host`, and mesh-sdk as the controller's sibling at the commit the controller's
// go.mod pins. In a mesh where either module has no source repository nothing is // go.mod pins. Each repository's tests read these siblings, and the checker places exactly these
// (mesh-delivery's checker, besideRules, follows this list): mesh-controller's read mesh-catalog,
// mesh-host and mesh-sdk; mesh-host's read mesh-catalog, mesh-sdk and mesh-controller (its step-nine
// test reads the controller's own manifest, novox/hq issue 1295); mesh-sdk's read mesh-catalog and
// mesh-host; mesh-catalog's read mesh-sdk and mesh-controller; mesh-delivery's read mesh-sdk and
// mesh-catalog. In a mesh where either module has no source repository nothing is
// cloned, and these tests fail loudly with "not beside this check": a cause in the setup, not in the // cloned, and these tests fail loudly with "not beside this check": a cause in the setup, not in the
// change. And in a delivery group that holds a mesh-catalog pull request, these tests read the // change. And in a delivery group that holds a mesh-catalog pull request, these tests read the
// catalogue's main, not the group's head. // catalogue's main, not the group's head.
+1
View File
@@ -0,0 +1 @@
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
+1
View File
@@ -0,0 +1 @@
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
+1
View File
@@ -0,0 +1 @@
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
+8 -1
View File
@@ -24,7 +24,11 @@ func Version() string {
// FromBuildInfo reads the SDK's version out of a build info. A dependency replaced by another module version is that // FromBuildInfo reads the SDK's version out of a build info. A dependency replaced by another module version is that
// version; replaced by a local path it is the path, never empty. When the SDK is the main module it is "(devel)", // version; replaced by a local path it is the path, never empty. When the SDK is the main module it is "(devel)",
// followed by a space and the vcs.revision when the build stamped one. Without the SDK in the build it is "unknown". // followed by a space and the vcs.revision when the build stamped one. An SDK command built from another module that
// requires the SDK (`go build git.novox.be/novox/mesh-sdk/go/cmd/mesh-compose` in a reader's tree, as mesh-build-go's
// compose job builds it: novox/hq ADR 0482 clause 5) has the SDK as its main module at the version that module pins,
// read from the module cache with no vcs stamp: it is that version, never a bare "(devel)" that names no SDK. Without
// the SDK in the build it is "unknown".
func FromBuildInfo(bi *debug.BuildInfo) string { func FromBuildInfo(bi *debug.BuildInfo) string {
if bi == nil { if bi == nil {
return "unknown" return "unknown"
@@ -35,6 +39,9 @@ func FromBuildInfo(bi *debug.BuildInfo) string {
return "(devel) " + s.Value return "(devel) " + s.Value
} }
} }
if v := bi.Main.Version; v != "" && v != "(devel)" {
return v
}
return "(devel)" return "(devel)"
} }
for _, d := range bi.Deps { for _, d := range bi.Deps {
+12
View File
@@ -42,6 +42,18 @@ func TestTheMainModuleIsDevelWithItsRevision(t *testing.T) {
} }
} }
// The shape go gives `go build <the SDK's command>` run in a module that requires the SDK (captured with go1.27 from
// mesh-compose built in a reader's tree at bc0c25e51052: "mod git.novox.be/novox/mesh-sdk/go v0.1.14-…-bc0c25e51052",
// no vcs settings): the SDK is the main module at the pinned version, which names the SDK the binary carries.
func TestAnSDKCommandBuiltFromAReaderIsThePinnedVersion(t *testing.T) {
bi := &debug.BuildInfo{Path: sdkPath + "/cmd/mesh-compose",
Main: debug.Module{Path: sdkPath, Version: "v0.1.14-0.20261024112544-bc0c25e51052"},
Settings: []debug.BuildSetting{{Key: "-trimpath", Value: "true"}, {Key: "CGO_ENABLED", Value: "0"}}}
if got := FromBuildInfo(bi); got != "v0.1.14-0.20261024112544-bc0c25e51052" {
t.Fatalf("got %q", got)
}
}
func TestNoBuildInfoIsUnknown(t *testing.T) { func TestNoBuildInfoIsUnknown(t *testing.T) {
if got := FromBuildInfo(nil); got != "unknown" { if got := FromBuildInfo(nil); got != "unknown" {
t.Fatalf("got %q", got) t.Fatalf("got %q", got)