Pack a build's output like the build agent and keep the steps state's shape (issues 1471, 1479), part 7 of 7
This commit is contained in:
@@ -0,0 +1,100 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// Pack tars and gzips a directory the way the build agent does, byte for byte, and answers the archive and its
|
||||
// digest, "sha256:" and the hex of the archive's bytes (issue 1479; novox/hq ADR 0405 §7, ADR 0453 §7). It is a
|
||||
// copy of mesh-controller's internal/builder/builder.go pack (origin/main 073ccc29, :1351-1425), which stays
|
||||
// where it is until the build agent goes: a build module's digest is compared with the build agent's for one
|
||||
// fingerprint, so the bytes must be equal.
|
||||
//
|
||||
// Deterministically: regular files only (a symlink or a device is refused), sorted relative paths with
|
||||
// slashes, each header carrying its name, permission bits and size and nothing else, the gzip default writer.
|
||||
func Pack(root string) (archive []byte, digest string, err error) {
|
||||
raw, err := pack(root)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
sum := sha256.Sum256(raw)
|
||||
return raw, "sha256:" + hex.EncodeToString(sum[:]), nil
|
||||
}
|
||||
|
||||
func pack(root string) ([]byte, error) {
|
||||
info, err := os.Stat(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return nil, fmt.Errorf("%s is not a directory", root)
|
||||
}
|
||||
|
||||
var paths []string
|
||||
err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.IsDir() || !info.Mode().IsRegular() {
|
||||
// Only files. A symlink or a device in an archive is refused by the host that unpacks
|
||||
// it, so putting one in would build something that cannot be applied.
|
||||
if !info.IsDir() && !info.Mode().IsRegular() {
|
||||
return fmt.Errorf("%s is neither a file nor a directory, and an archive carries "+
|
||||
"only those", path)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
paths = append(paths, path)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// filepath.Walk walks in lexical order; the sort is kept as the build agent keeps it, since an archive
|
||||
// whose digest changes because the traversal did is a silent failure.
|
||||
sort.Strings(paths)
|
||||
|
||||
var out bytes.Buffer
|
||||
zipped := gzip.NewWriter(&out)
|
||||
writer := tar.NewWriter(zipped)
|
||||
for _, path := range paths {
|
||||
body, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
relative, err := filepath.Rel(root, path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mode := int64(info.Mode().Perm())
|
||||
if err := writer.WriteHeader(&tar.Header{
|
||||
Name: filepath.ToSlash(relative), Mode: mode, Size: int64(len(body)),
|
||||
Typeflag: tar.TypeReg,
|
||||
// Everything else left at its zero value on purpose: no times, owners or names.
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := writer.Write(body); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if err := writer.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := zipped.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out.Bytes(), nil
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// fixture writes three files in two directories (modes 0644 and 0755).
|
||||
func fixture(t *testing.T) string {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
for _, f := range []struct {
|
||||
p string
|
||||
mode os.FileMode
|
||||
body string
|
||||
}{{"a/x.txt", 0o644, "x\n"}, {"a/run.sh", 0o755, "#!/bin/sh\necho hi\n"}, {"b/c/y.txt", 0o644, "why\n"}} {
|
||||
path := filepath.Join(root, f.p)
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(f.body), f.mode); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Chmod(path, f.mode); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return root
|
||||
}
|
||||
|
||||
// 1479/AC1: the archive equals the bytes mesh-controller's pack makes for this tree. testdata/pack-golden.tar.gz
|
||||
// was made once by running mesh-controller internal/builder/builder.go pack (origin/main 073ccc29, :1351-1425)
|
||||
// on this fixture (fixture above, modes set by chmod).
|
||||
func TestPackEqualsTheBuildAgentsArchive(t *testing.T) {
|
||||
golden, err := os.ReadFile("testdata/pack-golden.tar.gz")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
archive, digest, err := Pack(fixture(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(archive, golden) {
|
||||
t.Errorf("the archive is %d bytes, the build agent's %d, and they differ", len(archive), len(golden))
|
||||
}
|
||||
sum := sha256.Sum256(golden)
|
||||
if digest != "sha256:"+hex.EncodeToString(sum[:]) {
|
||||
t.Errorf("digest = %s", digest)
|
||||
}
|
||||
}
|
||||
|
||||
// 1479/AC2: touching every file's time changes nothing.
|
||||
func TestPackDigestIgnoresFileTimes(t *testing.T) {
|
||||
root := fixture(t)
|
||||
_, first, err := Pack(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
later := time.Now().Add(48 * time.Hour)
|
||||
filepath.Walk(root, func(p string, _ os.FileInfo, _ error) error { return os.Chtimes(p, later, later) })
|
||||
if _, second, err := Pack(root); err != nil || second != first {
|
||||
t.Errorf("digests %s and %s (%v)", first, second, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
//go:build unix
|
||||
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// 1479/AC3: anything but a file or a directory is refused, naming it. The rule of this repository is that no
|
||||
// test makes a symlink, so the non-regular entry is a named pipe: Pack refuses a symlink on the same branch
|
||||
// (filepath.Walk reports it by lstat as not regular).
|
||||
func TestPackRefusesAnEntryThatIsNotAFile(t *testing.T) {
|
||||
root := fixture(t)
|
||||
pipe := filepath.Join(root, "a", "pipe")
|
||||
if err := syscall.Mkfifo(pipe, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _, err := Pack(root)
|
||||
if err == nil || !strings.Contains(err.Error(), pipe) {
|
||||
t.Errorf("Pack = %v, want a refusal naming %s", err, pipe)
|
||||
}
|
||||
}
|
||||
BIN
Binary file not shown.
@@ -0,0 +1,116 @@
|
||||
package delivery
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// StepsState is the name of the state a build module keeps its long steps in; its bucket is `<module>_steps`, built
|
||||
// by StepsBucket (novox/hq ADR 0469 clause 2; issue 1471). A tool call is cut at 30 s, so a fetch, a test level or a
|
||||
// build answers at once with a step id and writes its progress and its end to the key `<node>.<step id>`.
|
||||
const StepsState = "steps"
|
||||
|
||||
// StepsBucket is the bucket of a build module's steps.
|
||||
func StepsBucket(module string) string { return module + "_" + StepsState }
|
||||
|
||||
// The states of a step. A step ends in passed, failed or refused; its key is written for each of them, and that
|
||||
// write is the event the step's reader waits on.
|
||||
const (
|
||||
StepRunning = "running"
|
||||
StepPassed = "passed"
|
||||
StepFailed = "failed"
|
||||
StepRefused = "refused"
|
||||
)
|
||||
|
||||
var stepIDOK = regexp.MustCompile(`^[a-z0-9]+$`)
|
||||
|
||||
func checkStepID(id string) error {
|
||||
if !stepIDOK.MatchString(id) {
|
||||
return fmt.Errorf("delivery: the step id %q is not lower-case letters and digits", id)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// StepKey is the key of a node's step: `<node>.<id>`. It refuses a node that is not a name and an id that is not
|
||||
// lower-case letters and digits (one key token, never a dot).
|
||||
func StepKey(node, id string) (string, error) {
|
||||
if err := checkName("node", node); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := checkStepID(id); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return node + "." + id, nil
|
||||
}
|
||||
|
||||
// ParseStepKey is the inverse of StepKey.
|
||||
func ParseStepKey(key string) (node, id string, err error) {
|
||||
node, id, ok := strings.Cut(key, ".")
|
||||
if !ok {
|
||||
return "", "", fmt.Errorf("delivery: %q is not a step key, <node>.<id>", key)
|
||||
}
|
||||
if _, err := StepKey(node, id); err != nil {
|
||||
return "", "", fmt.Errorf("delivery: %q is not a step key: %w", key, err)
|
||||
}
|
||||
return node, id, nil
|
||||
}
|
||||
|
||||
// Step is the value of a steps key: which tool runs on which node, how far it is, when it started and ended (UTC, as
|
||||
// the toolchains' At), why it was refused or failed, and what it returned.
|
||||
type Step struct {
|
||||
V int `json:"v"`
|
||||
ID string `json:"id"`
|
||||
Tool string `json:"tool"`
|
||||
Node string `json:"node"`
|
||||
State string `json:"state"`
|
||||
Started time.Time `json:"started"`
|
||||
Ended time.Time `json:"ended,omitzero"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
Result json.RawMessage `json:"result,omitempty"`
|
||||
}
|
||||
|
||||
func (s Step) check() error {
|
||||
if err := checkStepID(s.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := checkName("node", s.Node); err != nil {
|
||||
return err
|
||||
}
|
||||
if s.Tool == "" {
|
||||
return fmt.Errorf("delivery: a step needs its tool")
|
||||
}
|
||||
switch s.State {
|
||||
case StepRunning, StepPassed, StepFailed, StepRefused:
|
||||
default:
|
||||
return fmt.Errorf("delivery: the step state %q is not running, passed, failed or refused", s.State)
|
||||
}
|
||||
if len(s.Result) > 0 && !json.Valid(s.Result) {
|
||||
return fmt.Errorf("delivery: the step's result is not JSON")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// EncodeStep is s as JSON at Version; it refuses an id that is not one key token, a node that is not a name, an empty
|
||||
// tool, a state outside the four and a result that is not JSON.
|
||||
func EncodeStep(s Step) ([]byte, error) {
|
||||
if err := s.check(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.V = Version
|
||||
return json.Marshal(s)
|
||||
}
|
||||
|
||||
// DecodeStep reads a steps value; it refuses an unknown version (ErrUnknownVersion) and what EncodeStep refuses.
|
||||
func DecodeStep(data []byte) (Step, error) {
|
||||
var s Step
|
||||
if err := decode(data, &s); err != nil {
|
||||
return Step{}, err
|
||||
}
|
||||
if err := s.check(); err != nil {
|
||||
return Step{}, err
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package delivery
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// 1471/AC4: an encoded step with V 99 is refused with ErrUnknownVersion; so is a state 'done'.
|
||||
func TestDecodeStepRefusesAnUnknownVersionAndAState(t *testing.T) {
|
||||
good, err := EncodeStep(Step{ID: "k3j2", Tool: "fetch", Node: "ace", State: StepRunning, Started: time.Date(2026, 10, 20, 12, 0, 0, 0, time.UTC)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := DecodeStep(good); err != nil {
|
||||
t.Fatalf("a good step: %v", err)
|
||||
}
|
||||
v99 := strings.Replace(string(good), `"v":1`, `"v":99`, 1)
|
||||
if _, err := DecodeStep([]byte(v99)); !errors.Is(err, ErrUnknownVersion) {
|
||||
t.Errorf("version 99: %v, want ErrUnknownVersion", err)
|
||||
}
|
||||
done := strings.Replace(string(good), `"running"`, `"done"`, 1)
|
||||
if _, err := DecodeStep([]byte(done)); err == nil || errors.Is(err, ErrUnknownVersion) {
|
||||
t.Errorf("state done: %v, want a refusal of the state", err)
|
||||
}
|
||||
if _, err := EncodeStep(Step{ID: "a", Tool: "fetch", Node: "ace", State: "done"}); err == nil {
|
||||
t.Error("EncodeStep wrote the state done")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStepKeyIsNodeDotIDAndRefusesWhatIsNotOneToken(t *testing.T) {
|
||||
key, err := StepKey("ace", "k3j2")
|
||||
if err != nil || key != "ace.k3j2" {
|
||||
t.Fatalf("StepKey = %q, %v", key, err)
|
||||
}
|
||||
node, id, err := ParseStepKey(key)
|
||||
if err != nil || node != "ace" || id != "k3j2" {
|
||||
t.Errorf("ParseStepKey = %q, %q, %v", node, id, err)
|
||||
}
|
||||
for _, id := range []string{"", "a.b", "A", "a-b", "a b", "*"} {
|
||||
if _, err := StepKey("ace", id); err == nil {
|
||||
t.Errorf("StepKey accepted the id %q", id)
|
||||
}
|
||||
}
|
||||
for _, key := range []string{"ace", "ace.", "ace.a.b"} {
|
||||
if _, _, err := ParseStepKey(key); err == nil {
|
||||
t.Errorf("ParseStepKey accepted %q", key)
|
||||
}
|
||||
}
|
||||
if StepsBucket("mesh-build-go") != "mesh-build-go_steps" {
|
||||
t.Errorf("bucket = %q", StepsBucket("mesh-build-go"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestStepEncodingKeepsItsFieldsAndOmitsAnOpenEnd(t *testing.T) {
|
||||
at := time.Date(2026, 10, 20, 12, 0, 0, 0, time.UTC)
|
||||
raw, _ := EncodeStep(Step{ID: "a1", Tool: "fetch", Node: "ace", State: StepRunning, Started: at})
|
||||
want := `{"v":1,"id":"a1","tool":"fetch","node":"ace","state":"running","started":"2026-10-20T12:00:00Z"}`
|
||||
if string(raw) != want {
|
||||
t.Errorf("running = %s", raw)
|
||||
}
|
||||
raw, _ = EncodeStep(Step{ID: "a1", Tool: "fetch", Node: "ace", State: StepPassed, Started: at, Ended: at, Result: []byte(`{"sha":"x"}`)})
|
||||
got, err := DecodeStep(raw)
|
||||
if err != nil || got.State != StepPassed || string(got.Result) != `{"sha":"x"}` || !got.Ended.Equal(at) {
|
||||
t.Errorf("passed = %+v, %v", got, err)
|
||||
}
|
||||
if _, err := EncodeStep(Step{ID: "a1", Tool: "fetch", Node: "ace", State: StepPassed, Result: []byte(`{`)}); err == nil {
|
||||
t.Error("a result that is not JSON was written")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user