From 423b24d4ee8bbd5fe48641c4cf7e9e30b3cf9fa5 Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Wed, 14 Oct 2026 18:45:49 +0000 Subject: [PATCH] Bring F3 up to its release branch, with both sides' store tests (part 6 of 14) Co-Authored-By: Claude Opus 5.5 --- internal/builder/lend_test.go | 140 ++++++++++++++++++++++++++ internal/builder/reuse.go | 7 ++ internal/builder/reuse_test.go | 16 ++- internal/builder/template.go | 94 +++++++++++++++++ internal/builder/template_test.go | 136 +++++++++++++++++++++++++ internal/catalogue/build.go | 24 +++++ internal/catalogue/co_located_test.go | 15 ++- 7 files changed, 427 insertions(+), 5 deletions(-) create mode 100644 internal/builder/lend_test.go create mode 100644 internal/builder/template.go create mode 100644 internal/builder/template_test.go diff --git a/internal/builder/lend_test.go b/internal/builder/lend_test.go new file mode 100644 index 00000000..ff92a998 --- /dev/null +++ b/internal/builder/lend_test.go @@ -0,0 +1,140 @@ +package builder + +import ( + "context" + "encoding/json" + "net/http/httptest" + "runtime" + "strings" + "testing" +) + +// A built image's bases are linked into its repository before it is pushed (novox/hq issue 694): a +// runtime keeping containerd's image store may not hold a base layer it never had to fetch, and its push +// asks this machine only for what the repository lacks. + +// aBaseForThisMachine puts an index into the registry under `repository`: one manifest for this +// machine's platform, one for another, and answers the index's digest and each manifest's blobs. +func aBaseForThisMachine(t *testing.T, dst *aRegistryOfRepositories, repository string) (string, []string, []string) { + t.Helper() + put := func(arch string) (string, []string) { + config := []byte(`{"architecture":"` + arch + `"}`) + layer := []byte(arch + " layer bytes") + for _, b := range [][]byte{config, layer} { + dst.blobs[digestOf(b)] = b + dst.link(repository, digestOf(b)) + } + m, _ := json.Marshal(map[string]any{ + "schemaVersion": 2, "mediaType": mediaManifestOCI, + "config": map[string]any{"mediaType": "application/vnd.oci.image.config.v1+json", "digest": digestOf(config), "size": len(config)}, + "layers": []map[string]any{{"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": digestOf(layer), "size": len(layer)}}, + }) + dst.manifests[repository+"@"+digestOf(m)] = m + return digestOf(m), []string{digestOf(config), digestOf(layer)} + } + other := "s390x" + if runtime.GOARCH == other { + other = "amd64" + } + mine, mineBlobs := put(runtime.GOARCH) + theirs, theirBlobs := put(other) + index, _ := json.Marshal(map[string]any{ + "schemaVersion": 2, "mediaType": mediaIndexOCI, + "manifests": []map[string]any{ + {"mediaType": mediaManifestOCI, "digest": mine, "size": 1, "platform": map[string]string{"os": runtime.GOOS, "architecture": runtime.GOARCH}}, + {"mediaType": mediaManifestOCI, "digest": theirs, "size": 1, "platform": map[string]string{"os": "linux", "architecture": other}}, + }, + }) + dst.manifests[repository+"@"+digestOf(index)] = index + return digestOf(index), mineBlobs, theirBlobs +} + +func TestABuildsBaseIsLinkedIntoItsRepositoryBeforeThePush(t *testing.T) { + dst := newRegistryOfRepositories() + server := httptest.NewServer(dst.handler()) + defer server.Close() + address := strings.TrimPrefix(server.URL, "http://") + r := Registry{Address: address} + + const from = "upstream/docker.io/library/golang" + index, mine, theirs := aBaseForThisMachine(t, dst, from) + bases := []string{ + address + "/" + from + "@" + index, + // Not bases this registry holds: a package version, a tag, an image held nowhere here. + "@novox/mesh-sdk@1.2.3", + "golang:1-alpine", + "elsewhere.invalid/thing@sha256:" + strings.Repeat("e", 64), + } + n, err := r.LendBases(context.Background(), "mesh-tools-go/build", bases) + if err != nil { + t.Fatal(err) + } + // Every blob of this machine's platform is in the image's repository, so a push asks this machine's + // store for none of them; another platform's are not, since nothing built here stands on them. + for _, blob := range mine { + if !dst.links["mesh-tools-go/build"][blob] { + t.Fatalf("the base's blob %s is not in the image's repository", blob) + } + } + for _, blob := range theirs { + if dst.links["mesh-tools-go/build"][blob] { + t.Fatalf("another platform's blob %s was linked", blob) + } + } + if n != len(mine) || dst.mounts != len(mine) || dst.uploads != 0 { + t.Fatalf("%d linked, %d mounts, %d uploads; want %d mounts and no bytes moved", n, dst.mounts, dst.uploads, len(mine)) + } + + // Asked again, nothing is mounted twice. + if n, err := r.LendBases(context.Background(), "mesh-tools-go/build", bases); err != nil || n != 0 { + t.Fatalf("a second lending mounted %d: %v", n, err) + } +} + +// A registry that will not mount is said, not hidden: the push is still tried, from the store. +func TestABaseTheRegistryWillNotMountIsAnError(t *testing.T) { + dst := newRegistryOfRepositories() + dst.noMount = true + server := httptest.NewServer(dst.handler()) + defer server.Close() + address := strings.TrimPrefix(server.URL, "http://") + index, _, _ := aBaseForThisMachine(t, dst, "upstream/thing") + _, err := Registry{Address: address}.LendBases(context.Background(), "hello/server", + []string{address + "/upstream/thing@" + index}) + if err == nil || !strings.Contains(err.Error(), "did not mount") { + t.Fatalf("a refused mount passed silently: %v", err) + } +} + +// lending records what a build lent, and when, against the push. +type lending struct { + *mirroring + order []string +} + +func (l *lending) LendBases(_ context.Context, repository string, bases []string) (int, error) { + l.order = append(l.order, "lend "+repository+" "+strings.Join(bases, " ")) + return len(bases), nil +} + +func (l *lending) PublishImage(ctx context.Context, localTag, repository string) (string, error) { + l.order = append(l.order, "push "+repository) + return l.mirroring.PublishImage(ctx, localTag, repository) +} + +func TestAnImageIsPushedOnlyAfterItsBasesAreLent(t *testing.T) { + r, workspace := aRepository(t, anImage, map[string]string{"modules/bus/Dockerfile": "ARG BASE\nFROM ${BASE}"}) + r.contents["modules/bus/module.json"] = anImage + l := &lending{mirroring: &mirroring{recorded: r, at: "127.0.0.1:5000"}} + if _, err := Build(context.Background(), r.run, l, "https://forge.invalid/catalogue.git", "modules/bus", "", workspace, + nil, Npmrc{}, GitCredential{}, nil); err != nil { + t.Fatal(err) + } + want := []string{ + "lend bus/server 127.0.0.1:5000/upstream/docker.io/vendor/server@sha256:" + strings.Repeat("1", 64), + "push bus/server", + } + if strings.Join(l.order, "\n") != strings.Join(want, "\n") { + t.Fatalf("the build did:\n%s\nwant:\n%s", strings.Join(l.order, "\n"), strings.Join(want, "\n")) + } +} diff --git a/internal/builder/reuse.go b/internal/builder/reuse.go index 04a09ce0..2def775d 100644 --- a/internal/builder/reuse.go +++ b/internal/builder/reuse.go @@ -257,6 +257,13 @@ func (s scratchPublisher) PublishArchive(ctx context.Context, repository string, return s.to.PublishArchive(ctx, ScratchPrefix+repository, body, digest) } +func (s scratchPublisher) LendBases(ctx context.Context, repository string, bases []string) (int, error) { + if l, can := s.to.(BaseLender); can { + return l.LendBases(ctx, ScratchPrefix+repository, bases) + } + return 0, nil +} + func (s scratchPublisher) MirrorImage(ctx context.Context, from, repository string) (string, error) { m, can := s.to.(Mirrorer) if !can { diff --git a/internal/builder/reuse_test.go b/internal/builder/reuse_test.go index e3e4ec24..3b3d86e4 100644 --- a/internal/builder/reuse_test.go +++ b/internal/builder/reuse_test.go @@ -73,8 +73,14 @@ const reusedModule = `{"module":"meshboard","version":"1", // aModuleAt is a repository holding the module under mod/, whose directory git names as tree, on its trunk. func aModuleAt(t *testing.T, tree string) (*recorded, Runner, string) { t.Helper() - r := &recorded{tree: tree, contents: map[string]string{"mod/" + ManifestName: reusedModule, - "mod/Dockerfile": "FROM scratch", "mod/files/theme.conf": "dark"}} + return aModuleWith(t, tree, map[string]string{"mod/" + ManifestName: reusedModule, + "mod/Dockerfile": "FROM scratch", "mod/files/theme.conf": "dark"}) +} + +// aModuleWith is aModuleAt with the repository's files given. +func aModuleWith(t *testing.T, tree string, contents map[string]string) (*recorded, Runner, string) { + t.Helper() + r := &recorded{tree: tree, contents: contents} run := func(ctx context.Context, dir, name string, args ...string) (string, error) { if name == "git" && len(args) > 0 && args[0] == "symbolic-ref" { r.ran = append(r.ran, name+" "+strings.Join(args, " ")) @@ -100,6 +106,12 @@ var aPull = PullRef{Owner: "novox", Repo: "meshboard", Number: 7} func aCheckBuilt(t *testing.T, scratch *aScratch, tree string) CheckBuilt { t.Helper() r, run, workspace := aModuleAt(t, tree) + return aCheckBuiltBy(t, scratch, r, run, workspace) +} + +// aCheckBuiltBy is aCheckBuilt of the repository given. +func aCheckBuiltBy(t *testing.T, scratch *aScratch, r *recorded, run Runner, workspace string) CheckBuilt { + t.Helper() spec := CheckSpec{ID: "build-1", Owner: aPull.Owner, Repo: aPull.Repo, Number: aPull.Number, Builds: []CheckBuildSpec{{Module: "meshboard", Repository: "https://forge.invalid/meshboard.git", Path: "mod", Ref: "main"}}} diff --git a/internal/builder/template.go b/internal/builder/template.go new file mode 100644 index 00000000..1138b566 --- /dev/null +++ b/internal/builder/template.go @@ -0,0 +1,94 @@ +package builder + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "path/filepath" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// withTemplateTexts is what was built with each template's text, for a module whose files name templates +// (novox/hq ADR 0349 §2): its templates archive gains the text of every file in it, read back from the packed +// bytes, by its path inside the archive. Resolve reads them into the files' content. +// +// **Read from the archive, proven by its digest.** The module's files/ is packed again, as the build packs it, +// and the bytes are read only when they hash to the digest the build made or took: a fresh build's own archive, +// or a check's build a walk copied by digest without building (novox/hq ADR 0350 §2). Packing is deterministic, +// so one source packs to one digest; a tree that packs to another is not the archive, and its text is refused. +func withTemplateTexts(manifest catalogue.Manifest, within string, built []catalogue.Built) ([]catalogue.Built, error) { + if !manifest.Templated() { + return built, nil + } + name := catalogue.TemplatesArtifact(manifest) + at := -1 + for i, b := range built { + if b.Name == name { + at = i + } + } + if name == "" || at < 0 { + return nil, fmt.Errorf("%s names templates and built no archive of %s/ to read them from", + manifest.Module, catalogue.TemplatesDir) + } + body, err := pack(filepath.Join(within, catalogue.TemplatesDir)) + if err != nil { + return nil, fmt.Errorf("%s: packing %s/ to read its templates back failed: %w", manifest.Module, + catalogue.TemplatesDir, err) + } + sum := sha256.Sum256(body) + digest := "sha256:" + hex.EncodeToString(sum[:]) + if built[at].Digest != digest { + return nil, fmt.Errorf("%s: %s/ packs to the digest %s, and the archive %q built or taken is %s: the "+ + "templates are read only from the archive the mesh records", manifest.Module, catalogue.TemplatesDir, + digest, name, orNone(built[at].Digest)) + } + texts, err := unpackTexts(body) + if err != nil { + return nil, fmt.Errorf("%s: reading the archive %q back failed: %w", manifest.Module, name, err) + } + out := append([]catalogue.Built(nil), built...) + out[at].Texts = texts + return out, nil +} + +// unpackTexts is every file of a packed archive (pack), by its path inside it. +func unpackTexts(body []byte) (map[string]string, error) { + zipped, err := gzip.NewReader(bytes.NewReader(body)) + if err != nil { + return nil, err + } + defer zipped.Close() + reader := tar.NewReader(zipped) + texts := map[string]string{} + for { + header, err := reader.Next() + if errors.Is(err, io.EOF) { + return texts, nil + } + if err != nil { + return nil, err + } + if header.Typeflag != tar.TypeReg { + continue + } + text, err := io.ReadAll(reader) + if err != nil { + return nil, err + } + texts[header.Name] = string(text) + } +} + +func orNone(digest string) string { + if digest == "" { + return "no digest" + } + return digest +} diff --git a/internal/builder/template_test.go b/internal/builder/template_test.go new file mode 100644 index 00000000..40ff2777 --- /dev/null +++ b/internal/builder/template_test.go @@ -0,0 +1,136 @@ +package builder + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// A module's files kept as template files in its repository (novox/hq ADR 0349 §2, design 57 package T1, issue +// 654): the build packs files/ into the module's templates archive and reads each template's text back from the +// packed bytes into the file's content, so the manifest the mesh holds is the one it would hold for the text inline. + +const templatedModule = `{"module":"meshboard","version":"1", + "build":{"artifacts":[ + {"name":"server","kind":"image","from":"Dockerfile"}, + {"name":"files","kind":"archive","from":"files"}]}, + "resources":[ + {"id":"svc","type":"container","name":"meshboard","artifact":"server"}, + {"id":"conf","type":"file","path":"/etc/meshboard/meshboard.conf","template":"files/meshboard.conf"}, + {"id":"deeper","type":"file","path":"/etc/meshboard/deeper.conf","template":"files/a/deeper.conf"}]}` + +var templatedFiles = map[string]string{ + "Dockerfile": "FROM scratch", "files/meshboard.conf": "listen = 8080\n\"quoted\" é\n", + "files/a/deeper.conf": "deeper\n", "files/unused.conf": "packed, named by nothing", +} + +func contentOf(t *testing.T, m catalogue.Manifest, id string) string { + t.Helper() + for _, r := range m.Resources { + if r["id"] == id { + if _, has := r[catalogue.TemplateField]; has { + t.Fatalf("%s still names its template in the built manifest: %v", id, r) + } + text, ok := r["content"].(string) + if !ok { + t.Fatalf("%s carries no content: %v", id, r) + } + return text + } + } + t.Fatalf("no resource %s in %v", id, m.Resources) + return "" +} + +func TestABuildReadsEachTemplateBackFromItsArchive(t *testing.T) { + r, workspace := aRepository(t, templatedModule, templatedFiles) + got, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", workspace, nil, + Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if text := contentOf(t, got.Manifest, "conf"); text != templatedFiles["files/meshboard.conf"] { + t.Fatalf("the template's text is %q, not the file's", text) + } + if text := contentOf(t, got.Manifest, "deeper"); text != templatedFiles["files/a/deeper.conf"] { + t.Fatalf("the template's text is %q, not the file's", text) + } + if r.archives["meshboard/files"] == "" { + t.Fatalf("the templates archive was not published: %v", r.archives) + } + if problem := catalogue.HeldTemplateProblem(got.Manifest); problem != "" { + t.Fatal(problem) + } +} + +func TestABuildWhoseTemplateIsMissingFailsNamingIt(t *testing.T) { + files := map[string]string{} + for k, v := range templatedFiles { + files[k] = v + } + delete(files, "files/a/deeper.conf") + r, workspace := aRepository(t, templatedModule, files) + _, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", workspace, nil, + Npmrc{}, GitCredential{}, nil) + if err == nil || !strings.Contains(err.Error(), "files/a/deeper.conf") { + t.Fatalf("a build whose template is missing did not fail naming it: %v", err) + } +} + +// A walk that takes its check's build copies the archive by digest and builds nothing: the templates are read back +// from the tree packed again, which is the archive only when it packs to the digest taken. +func TestAWalkTakingAChecksBuildReadsTheTemplatesBackToo(t *testing.T) { + withVersion(t, "builder-1") + contents := map[string]string{"mod/" + ManifestName: templatedModule} + for k, v := range templatedFiles { + contents["mod/"+k] = v + } + scratch := newScratch() + r, run, workspace := aModuleWith(t, "tree-one", contents) + checked := aCheckBuiltBy(t, scratch, r, run, workspace) + + r, run, workspace = aModuleWith(t, "tree-one", contents) + got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: named(checked)}, + "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if got.Reused == nil || len(r.archives) > 0 { + t.Fatalf("the walk built again: reused %+v, why %q, published %v", got.Reused, got.Why, r.archives) + } + if text := contentOf(t, got.Manifest, "conf"); text != templatedFiles["files/meshboard.conf"] { + t.Fatalf("the template's text is %q, not the file's", text) + } +} + +// The tree packed again must be the archive taken; text read from anything else is not the archive's. +func TestTemplateTextsAreRefusedWhenTheTreeDoesNotPackToTheDigestTaken(t *testing.T) { + m, err := catalogue.ParseManifest([]byte(templatedModule)) + if err != nil { + t.Fatal(err) + } + within := writeTree(t, t.TempDir(), templatedFiles) + built := []catalogue.Built{{Name: "files", Kind: catalogue.ArtifactArchive, Digest: "sha256:" + strings.Repeat("0", 64)}} + if _, err := withTemplateTexts(m, within, built); err == nil || !strings.Contains(err.Error(), "digest") { + t.Fatalf("text was read from a tree that does not pack to the archive taken: %v", err) + } +} + +// writeTree writes files under dir and answers dir. +func writeTree(t *testing.T, dir string, files map[string]string) string { + t.Helper() + for name, body := range files { + full := filepath.Join(dir, filepath.FromSlash(name)) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(full, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + return dir +} diff --git a/internal/catalogue/build.go b/internal/catalogue/build.go index 8341916b..6fa59eb0 100644 --- a/internal/catalogue/build.go +++ b/internal/catalogue/build.go @@ -35,6 +35,10 @@ type Built struct { // Launchers are, for a bundle in an interpreted language, the executable the build wrote beside // each entrypoint, by entrypoint (novox/hq ADR 0193): what the node's runtime starts to serve it. Launchers map[string]string + // Texts are, for the archive a module packs its templates into (TemplatesArtifact), each file's text as the + // build read it back from the packed archive, by its path inside it (novox/hq ADR 0349 §2). Never carried on + // the wire or kept: Resolve reads them into the files' content, which is what the mesh holds. + Texts map[string]string `json:"-"` } // Resolve fills a manifest's resources in from what was built. @@ -71,6 +75,26 @@ func (m Manifest) Resolve(built []Built) (Manifest, error) { m.Module, strings.Join(missing, " and "), oneOrOther(len(missing))) } + // **Each template's text, as the build read it back from its templates archive** (novox/hq ADR 0349 §2): into + // the file's content, so what the mesh holds and composes is what the same text inline would be. + if m.Templated() { + name := TemplatesArtifact(m) + texts := by[name].Texts + if name == "" || texts == nil { + return Manifest{}, fmt.Errorf("%s names templates, and the build carried no text read back from the "+ + "archive it packs %s/ into (%q): the builder that built it does not read templates", + m.Module, TemplatesDir, name) + } + filled, err := fillTemplates(m.Module, m.Resources, func(template string) (string, bool) { + t, ok := texts[strings.TrimPrefix(template, TemplatesDir+"/")] + return t, ok + }, fmt.Sprintf("the archive %q", name)) + if err != nil { + return Manifest{}, err + } + m.Resources = filled + } + out := m out.Build = nil out.Resources = nil diff --git a/internal/catalogue/co_located_test.go b/internal/catalogue/co_located_test.go index 3373936f..340a8138 100644 --- a/internal/catalogue/co_located_test.go +++ b/internal/catalogue/co_located_test.go @@ -51,6 +51,15 @@ func stepCA() Manifest { } } +// stepCAWithItsRoot is the authority whose root the operator supplies: the served key asks for it as +// `${setting:root}` (ADR 0155), the only way a setting reaches what a provider serves since a layer +// stopped replacing a served key (novox/hq issue 696; before it, the empty `root` was replaced whole). +func stepCAWithItsRoot() Manifest { + m := stepCA() + m.Serves["acme-ca"]["root"] = "${setting:root}" + return m +} + // routeProxy issues from that authority, so it must be told the root to verify it with. func routeProxy() Manifest { return Manifest{ @@ -78,7 +87,7 @@ func routeProxy() Manifest { // The consequence is silent and total: route-proxy wrote an empty CA bundle, fell back to the // system trust store, could not verify the internal authority, and no certificate was ever issued. func TestACoLocatedProvidersServedValuesReachItsConsumer(t *testing.T) { - r, err := Resolve(shelf(stepCA(), routeProxy()), + r, err := Resolve(shelf(stepCAWithItsRoot(), routeProxy()), []string{"step-ca", "route-proxy"}, reachable(), World{}) if err != nil { t.Fatal(err) @@ -106,7 +115,7 @@ func TestACoLocatedProvidersServedValuesReachItsConsumer(t *testing.T) { // And the binding file says the same thing, for a consumer that reads the binding rather than a // substituted placeholder. The two are one fact and must not be able to disagree. func TestACoLocatedConsumersBindingCarriesTheSettledValues(t *testing.T) { - r, err := Resolve(shelf(stepCA(), routeProxy()), + r, err := Resolve(shelf(stepCAWithItsRoot(), routeProxy()), []string{"step-ca", "route-proxy"}, reachable(), World{}) if err != nil { t.Fatal(err) @@ -152,7 +161,7 @@ func TestACoLocatedProviderIsSettledWhicheverWayItWasPulledIn(t *testing.T) { {"step-ca", "route-proxy"}, {"route-proxy", "step-ca"}, } { - r, err := Resolve(shelf(stepCA(), routeProxy()), assigned, reachable(), World{}) + r, err := Resolve(shelf(stepCAWithItsRoot(), routeProxy()), assigned, reachable(), World{}) if err != nil { t.Fatal(err) }