diff --git a/internal/builder/template.go b/internal/builder/template.go new file mode 100644 index 00000000..1138b566 --- /dev/null +++ b/internal/builder/template.go @@ -0,0 +1,94 @@ +package builder + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "path/filepath" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// withTemplateTexts is what was built with each template's text, for a module whose files name templates +// (novox/hq ADR 0349 §2): its templates archive gains the text of every file in it, read back from the packed +// bytes, by its path inside the archive. Resolve reads them into the files' content. +// +// **Read from the archive, proven by its digest.** The module's files/ is packed again, as the build packs it, +// and the bytes are read only when they hash to the digest the build made or took: a fresh build's own archive, +// or a check's build a walk copied by digest without building (novox/hq ADR 0350 §2). Packing is deterministic, +// so one source packs to one digest; a tree that packs to another is not the archive, and its text is refused. +func withTemplateTexts(manifest catalogue.Manifest, within string, built []catalogue.Built) ([]catalogue.Built, error) { + if !manifest.Templated() { + return built, nil + } + name := catalogue.TemplatesArtifact(manifest) + at := -1 + for i, b := range built { + if b.Name == name { + at = i + } + } + if name == "" || at < 0 { + return nil, fmt.Errorf("%s names templates and built no archive of %s/ to read them from", + manifest.Module, catalogue.TemplatesDir) + } + body, err := pack(filepath.Join(within, catalogue.TemplatesDir)) + if err != nil { + return nil, fmt.Errorf("%s: packing %s/ to read its templates back failed: %w", manifest.Module, + catalogue.TemplatesDir, err) + } + sum := sha256.Sum256(body) + digest := "sha256:" + hex.EncodeToString(sum[:]) + if built[at].Digest != digest { + return nil, fmt.Errorf("%s: %s/ packs to the digest %s, and the archive %q built or taken is %s: the "+ + "templates are read only from the archive the mesh records", manifest.Module, catalogue.TemplatesDir, + digest, name, orNone(built[at].Digest)) + } + texts, err := unpackTexts(body) + if err != nil { + return nil, fmt.Errorf("%s: reading the archive %q back failed: %w", manifest.Module, name, err) + } + out := append([]catalogue.Built(nil), built...) + out[at].Texts = texts + return out, nil +} + +// unpackTexts is every file of a packed archive (pack), by its path inside it. +func unpackTexts(body []byte) (map[string]string, error) { + zipped, err := gzip.NewReader(bytes.NewReader(body)) + if err != nil { + return nil, err + } + defer zipped.Close() + reader := tar.NewReader(zipped) + texts := map[string]string{} + for { + header, err := reader.Next() + if errors.Is(err, io.EOF) { + return texts, nil + } + if err != nil { + return nil, err + } + if header.Typeflag != tar.TypeReg { + continue + } + text, err := io.ReadAll(reader) + if err != nil { + return nil, err + } + texts[header.Name] = string(text) + } +} + +func orNone(digest string) string { + if digest == "" { + return "no digest" + } + return digest +} diff --git a/internal/builder/template_test.go b/internal/builder/template_test.go new file mode 100644 index 00000000..40ff2777 --- /dev/null +++ b/internal/builder/template_test.go @@ -0,0 +1,136 @@ +package builder + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// A module's files kept as template files in its repository (novox/hq ADR 0349 §2, design 57 package T1, issue +// 654): the build packs files/ into the module's templates archive and reads each template's text back from the +// packed bytes into the file's content, so the manifest the mesh holds is the one it would hold for the text inline. + +const templatedModule = `{"module":"meshboard","version":"1", + "build":{"artifacts":[ + {"name":"server","kind":"image","from":"Dockerfile"}, + {"name":"files","kind":"archive","from":"files"}]}, + "resources":[ + {"id":"svc","type":"container","name":"meshboard","artifact":"server"}, + {"id":"conf","type":"file","path":"/etc/meshboard/meshboard.conf","template":"files/meshboard.conf"}, + {"id":"deeper","type":"file","path":"/etc/meshboard/deeper.conf","template":"files/a/deeper.conf"}]}` + +var templatedFiles = map[string]string{ + "Dockerfile": "FROM scratch", "files/meshboard.conf": "listen = 8080\n\"quoted\" é\n", + "files/a/deeper.conf": "deeper\n", "files/unused.conf": "packed, named by nothing", +} + +func contentOf(t *testing.T, m catalogue.Manifest, id string) string { + t.Helper() + for _, r := range m.Resources { + if r["id"] == id { + if _, has := r[catalogue.TemplateField]; has { + t.Fatalf("%s still names its template in the built manifest: %v", id, r) + } + text, ok := r["content"].(string) + if !ok { + t.Fatalf("%s carries no content: %v", id, r) + } + return text + } + } + t.Fatalf("no resource %s in %v", id, m.Resources) + return "" +} + +func TestABuildReadsEachTemplateBackFromItsArchive(t *testing.T) { + r, workspace := aRepository(t, templatedModule, templatedFiles) + got, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", workspace, nil, + Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if text := contentOf(t, got.Manifest, "conf"); text != templatedFiles["files/meshboard.conf"] { + t.Fatalf("the template's text is %q, not the file's", text) + } + if text := contentOf(t, got.Manifest, "deeper"); text != templatedFiles["files/a/deeper.conf"] { + t.Fatalf("the template's text is %q, not the file's", text) + } + if r.archives["meshboard/files"] == "" { + t.Fatalf("the templates archive was not published: %v", r.archives) + } + if problem := catalogue.HeldTemplateProblem(got.Manifest); problem != "" { + t.Fatal(problem) + } +} + +func TestABuildWhoseTemplateIsMissingFailsNamingIt(t *testing.T) { + files := map[string]string{} + for k, v := range templatedFiles { + files[k] = v + } + delete(files, "files/a/deeper.conf") + r, workspace := aRepository(t, templatedModule, files) + _, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", workspace, nil, + Npmrc{}, GitCredential{}, nil) + if err == nil || !strings.Contains(err.Error(), "files/a/deeper.conf") { + t.Fatalf("a build whose template is missing did not fail naming it: %v", err) + } +} + +// A walk that takes its check's build copies the archive by digest and builds nothing: the templates are read back +// from the tree packed again, which is the archive only when it packs to the digest taken. +func TestAWalkTakingAChecksBuildReadsTheTemplatesBackToo(t *testing.T) { + withVersion(t, "builder-1") + contents := map[string]string{"mod/" + ManifestName: templatedModule} + for k, v := range templatedFiles { + contents["mod/"+k] = v + } + scratch := newScratch() + r, run, workspace := aModuleWith(t, "tree-one", contents) + checked := aCheckBuiltBy(t, scratch, r, run, workspace) + + r, run, workspace = aModuleWith(t, "tree-one", contents) + got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: named(checked)}, + "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if got.Reused == nil || len(r.archives) > 0 { + t.Fatalf("the walk built again: reused %+v, why %q, published %v", got.Reused, got.Why, r.archives) + } + if text := contentOf(t, got.Manifest, "conf"); text != templatedFiles["files/meshboard.conf"] { + t.Fatalf("the template's text is %q, not the file's", text) + } +} + +// The tree packed again must be the archive taken; text read from anything else is not the archive's. +func TestTemplateTextsAreRefusedWhenTheTreeDoesNotPackToTheDigestTaken(t *testing.T) { + m, err := catalogue.ParseManifest([]byte(templatedModule)) + if err != nil { + t.Fatal(err) + } + within := writeTree(t, t.TempDir(), templatedFiles) + built := []catalogue.Built{{Name: "files", Kind: catalogue.ArtifactArchive, Digest: "sha256:" + strings.Repeat("0", 64)}} + if _, err := withTemplateTexts(m, within, built); err == nil || !strings.Contains(err.Error(), "digest") { + t.Fatalf("text was read from a tree that does not pack to the archive taken: %v", err) + } +} + +// writeTree writes files under dir and answers dir. +func writeTree(t *testing.T, dir string, files map[string]string) string { + t.Helper() + for name, body := range files { + full := filepath.Join(dir, filepath.FromSlash(name)) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(full, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + return dir +} diff --git a/internal/inventory/template_test.go b/internal/inventory/template_test.go new file mode 100644 index 00000000..b9f58cc8 --- /dev/null +++ b/internal/inventory/template_test.go @@ -0,0 +1,25 @@ +package inventory + +import ( + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// The manifest the mesh holds carries each file's text (novox/hq ADR 0349 §2): one still naming a template — a +// build by a builder that predates templates, or a manifest handed over unbuilt — would compose a file with no +// text, and is refused by name. +func TestAManifestStillNamingATemplateIsNotRegistered(t *testing.T) { + inv := ForTest(t) + m := catalogue.Manifest{Module: "notifier", Version: "1", Resources: []map[string]any{ + {"id": "conf", "type": "file", "path": "/etc/notifier.conf", "template": "files/notifier.conf"}}} + err := inv.RegisterModule(t.Context(), m, Source{}) + if err == nil || !strings.Contains(err.Error(), "files/notifier.conf") { + t.Fatalf("a manifest naming a template was registered: %v", err) + } + m.Resources[0] = map[string]any{"id": "conf", "type": "file", "path": "/etc/notifier.conf", "content": "x"} + if err := inv.RegisterModule(t.Context(), m, Source{}); err != nil { + t.Fatalf("the same manifest with its text was refused: %v", err) + } +}