diff --git a/go/buildstep/pack.go b/go/buildstep/pack.go new file mode 100644 index 0000000..bab02b2 --- /dev/null +++ b/go/buildstep/pack.go @@ -0,0 +1,100 @@ +package buildstep + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "sort" +) + +// Pack tars and gzips a directory the way the build agent does, byte for byte, and answers the archive and its +// digest, "sha256:" and the hex of the archive's bytes (issue 1479; novox/hq ADR 0405 §7, ADR 0453 §7). It is a +// copy of mesh-controller's internal/builder/builder.go pack (origin/main 073ccc29, :1351-1425), which stays +// where it is until the build agent goes: a build module's digest is compared with the build agent's for one +// fingerprint, so the bytes must be equal. +// +// Deterministically: regular files only (a symlink or a device is refused), sorted relative paths with +// slashes, each header carrying its name, permission bits and size and nothing else, the gzip default writer. +func Pack(root string) (archive []byte, digest string, err error) { + raw, err := pack(root) + if err != nil { + return nil, "", err + } + sum := sha256.Sum256(raw) + return raw, "sha256:" + hex.EncodeToString(sum[:]), nil +} + +func pack(root string) ([]byte, error) { + info, err := os.Stat(root) + if err != nil { + return nil, err + } + if !info.IsDir() { + return nil, fmt.Errorf("%s is not a directory", root) + } + + var paths []string + err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() || !info.Mode().IsRegular() { + // Only files. A symlink or a device in an archive is refused by the host that unpacks + // it, so putting one in would build something that cannot be applied. + if !info.IsDir() && !info.Mode().IsRegular() { + return fmt.Errorf("%s is neither a file nor a directory, and an archive carries "+ + "only those", path) + } + return nil + } + paths = append(paths, path) + return nil + }) + if err != nil { + return nil, err + } + // filepath.Walk walks in lexical order; the sort is kept as the build agent keeps it, since an archive + // whose digest changes because the traversal did is a silent failure. + sort.Strings(paths) + + var out bytes.Buffer + zipped := gzip.NewWriter(&out) + writer := tar.NewWriter(zipped) + for _, path := range paths { + body, err := os.ReadFile(path) + if err != nil { + return nil, err + } + relative, err := filepath.Rel(root, path) + if err != nil { + return nil, err + } + info, err := os.Stat(path) + if err != nil { + return nil, err + } + mode := int64(info.Mode().Perm()) + if err := writer.WriteHeader(&tar.Header{ + Name: filepath.ToSlash(relative), Mode: mode, Size: int64(len(body)), + Typeflag: tar.TypeReg, + // Everything else left at its zero value on purpose: no times, owners or names. + }); err != nil { + return nil, err + } + if _, err := writer.Write(body); err != nil { + return nil, err + } + } + if err := writer.Close(); err != nil { + return nil, err + } + if err := zipped.Close(); err != nil { + return nil, err + } + return out.Bytes(), nil +} diff --git a/go/buildstep/pack_test.go b/go/buildstep/pack_test.go new file mode 100644 index 0000000..3f5282d --- /dev/null +++ b/go/buildstep/pack_test.go @@ -0,0 +1,69 @@ +package buildstep + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "os" + "path/filepath" + "testing" + "time" +) + +// fixture writes three files in two directories (modes 0644 and 0755). +func fixture(t *testing.T) string { + t.Helper() + root := t.TempDir() + for _, f := range []struct { + p string + mode os.FileMode + body string + }{{"a/x.txt", 0o644, "x\n"}, {"a/run.sh", 0o755, "#!/bin/sh\necho hi\n"}, {"b/c/y.txt", 0o644, "why\n"}} { + path := filepath.Join(root, f.p) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(f.body), f.mode); err != nil { + t.Fatal(err) + } + if err := os.Chmod(path, f.mode); err != nil { + t.Fatal(err) + } + } + return root +} + +// 1479/AC1: the archive equals the bytes mesh-controller's pack makes for this tree. testdata/pack-golden.tar.gz +// was made once by running mesh-controller internal/builder/builder.go pack (origin/main 073ccc29, :1351-1425) +// on this fixture (fixture above, modes set by chmod). +func TestPackEqualsTheBuildAgentsArchive(t *testing.T) { + golden, err := os.ReadFile("testdata/pack-golden.tar.gz") + if err != nil { + t.Fatal(err) + } + archive, digest, err := Pack(fixture(t)) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(archive, golden) { + t.Errorf("the archive is %d bytes, the build agent's %d, and they differ", len(archive), len(golden)) + } + sum := sha256.Sum256(golden) + if digest != "sha256:"+hex.EncodeToString(sum[:]) { + t.Errorf("digest = %s", digest) + } +} + +// 1479/AC2: touching every file's time changes nothing. +func TestPackDigestIgnoresFileTimes(t *testing.T) { + root := fixture(t) + _, first, err := Pack(root) + if err != nil { + t.Fatal(err) + } + later := time.Now().Add(48 * time.Hour) + filepath.Walk(root, func(p string, _ os.FileInfo, _ error) error { return os.Chtimes(p, later, later) }) + if _, second, err := Pack(root); err != nil || second != first { + t.Errorf("digests %s and %s (%v)", first, second, err) + } +} diff --git a/go/buildstep/pack_unix_test.go b/go/buildstep/pack_unix_test.go new file mode 100644 index 0000000..57a5052 --- /dev/null +++ b/go/buildstep/pack_unix_test.go @@ -0,0 +1,25 @@ +//go:build unix + +package buildstep + +import ( + "path/filepath" + "strings" + "syscall" + "testing" +) + +// 1479/AC3: anything but a file or a directory is refused, naming it. The rule of this repository is that no +// test makes a symlink, so the non-regular entry is a named pipe: Pack refuses a symlink on the same branch +// (filepath.Walk reports it by lstat as not regular). +func TestPackRefusesAnEntryThatIsNotAFile(t *testing.T) { + root := fixture(t) + pipe := filepath.Join(root, "a", "pipe") + if err := syscall.Mkfifo(pipe, 0o644); err != nil { + t.Fatal(err) + } + _, _, err := Pack(root) + if err == nil || !strings.Contains(err.Error(), pipe) { + t.Errorf("Pack = %v, want a refusal naming %s", err, pipe) + } +} diff --git a/go/buildstep/testdata/pack-golden.tar.gz b/go/buildstep/testdata/pack-golden.tar.gz new file mode 100644 index 0000000..e5ea0a8 Binary files /dev/null and b/go/buildstep/testdata/pack-golden.tar.gz differ diff --git a/go/delivery/steps.go b/go/delivery/steps.go new file mode 100644 index 0000000..61f8114 --- /dev/null +++ b/go/delivery/steps.go @@ -0,0 +1,116 @@ +package delivery + +import ( + "encoding/json" + "fmt" + "regexp" + "strings" + "time" +) + +// StepsState is the name of the state a build module keeps its long steps in; its bucket is `_steps`, built +// by StepsBucket (novox/hq ADR 0469 clause 2; issue 1471). A tool call is cut at 30 s, so a fetch, a test level or a +// build answers at once with a step id and writes its progress and its end to the key `.`. +const StepsState = "steps" + +// StepsBucket is the bucket of a build module's steps. +func StepsBucket(module string) string { return module + "_" + StepsState } + +// The states of a step. A step ends in passed, failed or refused; its key is written for each of them, and that +// write is the event the step's reader waits on. +const ( + StepRunning = "running" + StepPassed = "passed" + StepFailed = "failed" + StepRefused = "refused" +) + +var stepIDOK = regexp.MustCompile(`^[a-z0-9]+$`) + +func checkStepID(id string) error { + if !stepIDOK.MatchString(id) { + return fmt.Errorf("delivery: the step id %q is not lower-case letters and digits", id) + } + return nil +} + +// StepKey is the key of a node's step: `.`. It refuses a node that is not a name and an id that is not +// lower-case letters and digits (one key token, never a dot). +func StepKey(node, id string) (string, error) { + if err := checkName("node", node); err != nil { + return "", err + } + if err := checkStepID(id); err != nil { + return "", err + } + return node + "." + id, nil +} + +// ParseStepKey is the inverse of StepKey. +func ParseStepKey(key string) (node, id string, err error) { + node, id, ok := strings.Cut(key, ".") + if !ok { + return "", "", fmt.Errorf("delivery: %q is not a step key, .", key) + } + if _, err := StepKey(node, id); err != nil { + return "", "", fmt.Errorf("delivery: %q is not a step key: %w", key, err) + } + return node, id, nil +} + +// Step is the value of a steps key: which tool runs on which node, how far it is, when it started and ended (UTC, as +// the toolchains' At), why it was refused or failed, and what it returned. +type Step struct { + V int `json:"v"` + ID string `json:"id"` + Tool string `json:"tool"` + Node string `json:"node"` + State string `json:"state"` + Started time.Time `json:"started"` + Ended time.Time `json:"ended,omitzero"` + Reason string `json:"reason,omitempty"` + Result json.RawMessage `json:"result,omitempty"` +} + +func (s Step) check() error { + if err := checkStepID(s.ID); err != nil { + return err + } + if err := checkName("node", s.Node); err != nil { + return err + } + if s.Tool == "" { + return fmt.Errorf("delivery: a step needs its tool") + } + switch s.State { + case StepRunning, StepPassed, StepFailed, StepRefused: + default: + return fmt.Errorf("delivery: the step state %q is not running, passed, failed or refused", s.State) + } + if len(s.Result) > 0 && !json.Valid(s.Result) { + return fmt.Errorf("delivery: the step's result is not JSON") + } + return nil +} + +// EncodeStep is s as JSON at Version; it refuses an id that is not one key token, a node that is not a name, an empty +// tool, a state outside the four and a result that is not JSON. +func EncodeStep(s Step) ([]byte, error) { + if err := s.check(); err != nil { + return nil, err + } + s.V = Version + return json.Marshal(s) +} + +// DecodeStep reads a steps value; it refuses an unknown version (ErrUnknownVersion) and what EncodeStep refuses. +func DecodeStep(data []byte) (Step, error) { + var s Step + if err := decode(data, &s); err != nil { + return Step{}, err + } + if err := s.check(); err != nil { + return Step{}, err + } + return s, nil +} diff --git a/go/delivery/steps_test.go b/go/delivery/steps_test.go new file mode 100644 index 0000000..b9dbb7f --- /dev/null +++ b/go/delivery/steps_test.go @@ -0,0 +1,71 @@ +package delivery + +import ( + "errors" + "strings" + "testing" + "time" +) + +// 1471/AC4: an encoded step with V 99 is refused with ErrUnknownVersion; so is a state 'done'. +func TestDecodeStepRefusesAnUnknownVersionAndAState(t *testing.T) { + good, err := EncodeStep(Step{ID: "k3j2", Tool: "fetch", Node: "ace", State: StepRunning, Started: time.Date(2026, 10, 20, 12, 0, 0, 0, time.UTC)}) + if err != nil { + t.Fatal(err) + } + if _, err := DecodeStep(good); err != nil { + t.Fatalf("a good step: %v", err) + } + v99 := strings.Replace(string(good), `"v":1`, `"v":99`, 1) + if _, err := DecodeStep([]byte(v99)); !errors.Is(err, ErrUnknownVersion) { + t.Errorf("version 99: %v, want ErrUnknownVersion", err) + } + done := strings.Replace(string(good), `"running"`, `"done"`, 1) + if _, err := DecodeStep([]byte(done)); err == nil || errors.Is(err, ErrUnknownVersion) { + t.Errorf("state done: %v, want a refusal of the state", err) + } + if _, err := EncodeStep(Step{ID: "a", Tool: "fetch", Node: "ace", State: "done"}); err == nil { + t.Error("EncodeStep wrote the state done") + } +} + +func TestStepKeyIsNodeDotIDAndRefusesWhatIsNotOneToken(t *testing.T) { + key, err := StepKey("ace", "k3j2") + if err != nil || key != "ace.k3j2" { + t.Fatalf("StepKey = %q, %v", key, err) + } + node, id, err := ParseStepKey(key) + if err != nil || node != "ace" || id != "k3j2" { + t.Errorf("ParseStepKey = %q, %q, %v", node, id, err) + } + for _, id := range []string{"", "a.b", "A", "a-b", "a b", "*"} { + if _, err := StepKey("ace", id); err == nil { + t.Errorf("StepKey accepted the id %q", id) + } + } + for _, key := range []string{"ace", "ace.", "ace.a.b"} { + if _, _, err := ParseStepKey(key); err == nil { + t.Errorf("ParseStepKey accepted %q", key) + } + } + if StepsBucket("mesh-build-go") != "mesh-build-go_steps" { + t.Errorf("bucket = %q", StepsBucket("mesh-build-go")) + } +} + +func TestStepEncodingKeepsItsFieldsAndOmitsAnOpenEnd(t *testing.T) { + at := time.Date(2026, 10, 20, 12, 0, 0, 0, time.UTC) + raw, _ := EncodeStep(Step{ID: "a1", Tool: "fetch", Node: "ace", State: StepRunning, Started: at}) + want := `{"v":1,"id":"a1","tool":"fetch","node":"ace","state":"running","started":"2026-10-20T12:00:00Z"}` + if string(raw) != want { + t.Errorf("running = %s", raw) + } + raw, _ = EncodeStep(Step{ID: "a1", Tool: "fetch", Node: "ace", State: StepPassed, Started: at, Ended: at, Result: []byte(`{"sha":"x"}`)}) + got, err := DecodeStep(raw) + if err != nil || got.State != StepPassed || string(got.Result) != `{"sha":"x"}` || !got.Ended.Equal(at) { + t.Errorf("passed = %+v, %v", got, err) + } + if _, err := EncodeStep(Step{ID: "a1", Tool: "fetch", Node: "ace", State: StepPassed, Result: []byte(`{`)}); err == nil { + t.Error("a result that is not JSON was written") + } +}