diff --git a/internal/catalogue/build.go b/internal/catalogue/build.go new file mode 100644 index 0000000..31ff2b0 --- /dev/null +++ b/internal/catalogue/build.go @@ -0,0 +1,147 @@ +package catalogue + +import ( + "fmt" + "sort" + "strings" +) + +// Turning a manifest that names artifacts into one that names digests. +// +// **Two documents, deliberately.** The manifest in a repository says *this resource uses the +// archive called `config`*; the manifest the mesh holds says *this resource is sha256:…*. A digest +// is not knowable until something is built, so a repository carrying one would be a repository +// whose file is wrong the moment anybody edits anything — and the mesh would be pinning a value +// nobody could have checked. +// +// So the built manifest is **derived**, and the record of which commit it was derived from is what +// makes "is this current?" answerable without building (novox/hq ADR 0009). + +// Built is one artifact after it exists: where it is and what it hashes to. +type Built struct { + // Name is what the manifest called it. + Name string + // Kind is "image" or "archive". + Kind string + // Reference is what a machine uses to get it — an image reference for an image, a URL for an + // archive. Both already carry the digest for an image; an archive carries it separately. + Reference string + // Digest is "sha256:", for an archive. An image reference already ends in one. + Digest string +} + +// Resolve fills a manifest's resources in from what was built. +// +// Every resource naming an artifact is rewritten to name the thing itself, and the `artifact` key +// is removed — because it is a build-time word and the host has never heard of it. A resource +// naming an artifact nothing produced is refused: it would otherwise reach a machine with an +// empty image or an unpinned archive, which is the shape of failure that looks like success. +func (m Manifest) Resolve(built []Built) (Manifest, error) { + if m.Build == nil && len(built) == 0 { + return m, nil + } + + by := map[string]Built{} + for _, b := range built { + by[b.Name] = b + } + // Declared and not produced is a build that did not do what the manifest asked, and saying so + // here beats a machine reporting it later. + var missing []string + if m.Build != nil { + for _, a := range m.Build.Artifacts { + if _, ok := by[a.Name]; !ok { + missing = append(missing, a.Name) + } + } + } + if len(missing) > 0 { + sort.Strings(missing) + return Manifest{}, fmt.Errorf( + "%s says it builds %s and the build did not produce %s — the build did not do what "+ + "the manifest asked, which is a different fault from a resource asking for the "+ + "wrong thing", + m.Module, strings.Join(missing, " and "), oneOrOther(len(missing))) + } + + out := m + out.Build = nil + out.Resources = nil + for _, r := range m.Resources { + named, _ := r["artifact"].(string) + if named == "" { + out.Resources = append(out.Resources, r) + continue + } + artifact, ok := by[named] + if !ok { + return Manifest{}, fmt.Errorf( + "%s: %v uses the artifact %q, and this module builds no such thing", + m.Module, r["id"], named) + } + + filled := map[string]any{} + for k, v := range r { + filled[k] = v + } + delete(filled, "artifact") + switch artifact.Kind { + case ArtifactImage: + filled["image"] = artifact.Reference + case ArtifactArchive: + filled["source"] = artifact.Reference + filled["digest"] = artifact.Digest + default: + return Manifest{}, fmt.Errorf("%s: %q is a %q, and an artifact is %q or %q", + m.Module, named, artifact.Kind, ArtifactImage, ArtifactArchive) + } + out.Resources = append(out.Resources, filled) + } + return out, nil +} + +// checkBuild is the manifest's own account of what it builds. +func (b *Build) problems(module string) []string { + if b == nil { + return nil + } + var problems []string + seen := map[string]bool{} + for _, a := range b.Artifacts { + if a.Name == "" { + problems = append(problems, module+" builds an artifact with no name") + continue + } + if seen[a.Name] { + problems = append(problems, fmt.Sprintf( + "%s builds two artifacts called %q, and a resource naming it could mean either", + module, a.Name)) + } + seen[a.Name] = true + if a.Kind != ArtifactImage && a.Kind != ArtifactArchive { + problems = append(problems, fmt.Sprintf("%s: %q is a %q, and an artifact is %q or %q", + module, a.Name, a.Kind, ArtifactImage, ArtifactArchive)) + } + if a.From == "" { + problems = append(problems, fmt.Sprintf( + "%s: %q says nothing about what it is built from", module, a.Name)) + } + if strings.HasPrefix(a.From, "/") || strings.Contains(a.From, "..") { + // A build reads its own repository and nothing else. A path leaving it would make + // what gets built depend on whatever happens to be on the machine building it. + problems = append(problems, fmt.Sprintf( + "%s: %q is built from %q, which is outside its own repository", + module, a.Name, a.From)) + } + } + return problems +} + +// oneOrOther keeps the message readable for one artifact and for several, because a message that +// says "neither" about one thing reads as a bug in the message. +func oneOrOther(n int) string { + if n == 1 { + return "it" + } + return "them" +} diff --git a/internal/catalogue/build_test.go b/internal/catalogue/build_test.go new file mode 100644 index 0000000..cba3ed2 --- /dev/null +++ b/internal/catalogue/build_test.go @@ -0,0 +1,144 @@ +package catalogue + +import ( + "strings" + "testing" +) + +// The manifest in a repository and the manifest the mesh holds are two documents. +// +// A digest is not knowable until something is built, so a repository carrying one would be a +// repository whose file is wrong the moment anybody edits anything — and the mesh would be pinning +// a value nobody could have checked. + +func buildable() Manifest { + return Manifest{ + Module: "meshboard", Version: "1", + Build: &Build{Artifacts: []Artifact{ + {Name: "server", Kind: ArtifactImage, From: "Dockerfile"}, + {Name: "theme", Kind: ArtifactArchive, From: "files"}, + }}, + Resources: []map[string]any{ + {"id": "svc", "type": "container", "name": "meshboard", "artifact": "server"}, + {"id": "look", "type": "archive", "path": "/opt/meshboard", "artifact": "theme"}, + {"id": "dir", "type": "directory", "path": "/opt/meshboard"}, + }, + } +} + +func wasBuilt() []Built { + return []Built{ + {Name: "server", Kind: ArtifactImage, + Reference: "registry.invalid/meshboard@sha256:" + strings.Repeat("a", 64)}, + {Name: "theme", Kind: ArtifactArchive, + Reference: "https://store.invalid/theme.tar.gz", + Digest: "sha256:" + strings.Repeat("b", 64)}, + } +} + +func TestAResourceNamingAnArtifactBecomesOneNamingTheThing(t *testing.T) { + got, err := buildable().Resolve(wasBuilt()) + if err != nil { + t.Fatal(err) + } + for _, r := range got.Resources { + if r["artifact"] != nil { + // A build-time word the host has never heard of. Leaving it would be a field the + // strict decoder refuses on the machine, at the worst moment. + t.Fatalf("%v still names an artifact: %v", r["id"], r) + } + } + if got.Resources[0]["image"] != "registry.invalid/meshboard@sha256:"+strings.Repeat("a", 64) { + t.Fatalf("the image was not filled in: %v", got.Resources[0]) + } + if got.Resources[1]["source"] != "https://store.invalid/theme.tar.gz" || + got.Resources[1]["digest"] != "sha256:"+strings.Repeat("b", 64) { + t.Fatalf("the archive was not filled in: %v", got.Resources[1]) + } + // And a resource that names nothing is untouched. + if got.Resources[2]["path"] != "/opt/meshboard" || len(got.Resources[2]) != 3 { + t.Fatalf("an ordinary resource was changed: %v", got.Resources[2]) + } + // The built manifest carries no build section: it is the derived document, and something + // holding both would invite somebody to build from it again. + if got.Build != nil { + t.Fatal("the built manifest still says how to build itself") + } +} + +func TestAnArtifactDeclaredAndNotBuiltIsRefused(t *testing.T) { + // A build that did not do what the manifest asked. Said here rather than by a machine later + // reporting an empty image. + _, err := buildable().Resolve(wasBuilt()[:1]) + if err == nil { + t.Fatal("a manifest resolved with an artifact missing") + } + if !strings.Contains(err.Error(), "theme") { + t.Fatalf("the refusal does not name what is missing: %v", err) + } + // And it must blame the build rather than the resource. A resource asking for something that + // does not exist is a manifest fault; a build not producing what the manifest declared is a + // build fault, and telling somebody to fix the wrong one costs an afternoon. + if !strings.Contains(err.Error(), "the build did not produce") { + t.Fatalf("the refusal blames the wrong thing: %v", err) + } +} + +func TestAResourceNamingSomethingTheModuleDoesNotBuildIsRefused(t *testing.T) { + m := buildable() + m.Resources = append(m.Resources, + map[string]any{"id": "other", "type": "archive", "path": "/x", "artifact": "nothing"}) + if _, err := m.Resolve(wasBuilt()); err == nil { + t.Fatal("a resource naming an artifact nobody builds was accepted") + } +} + +func TestAModuleThatBuildsNothingIsOrdinary(t *testing.T) { + // Most of what a person installs is configuration. Requiring an empty build section would be + // a field that exists to be left blank. + m := Manifest{Module: "shell", Version: "1", Resources: []map[string]any{ + {"id": "rc", "type": "file", "path": "/etc/zsh/zshrc", "content": "setopt"}, + }} + got, err := m.Resolve(nil) + if err != nil { + t.Fatal(err) + } + if len(got.Resources) != 1 { + t.Fatalf("got %v", got.Resources) + } +} + +func TestTwoArtifactsWithOneNameAreRefused(t *testing.T) { + _, err := ParseManifest([]byte(`{"module":"a","version":"1","build":{"artifacts":[ + {"name":"x","kind":"image","from":"Dockerfile"}, + {"name":"x","kind":"archive","from":"files"}]}}`)) + if err == nil { + t.Fatal("two artifacts with one name were accepted") + } + if !strings.Contains(err.Error(), "could mean either") { + t.Fatalf("unhelpful refusal: %v", err) + } +} + +func TestAnArtifactBuiltFromOutsideItsRepositoryIsRefused(t *testing.T) { + // A build reads its own repository and nothing else. A path leaving it would make what gets + // built depend on whatever happens to be on the machine doing the building. + for _, from := range []string{"/etc/passwd", "../elsewhere"} { + _, err := ParseManifest([]byte(`{"module":"a","version":"1","build":{"artifacts":[ + {"name":"x","kind":"archive","from":"` + from + `"}]}}`)) + if err == nil { + t.Fatalf("%q was accepted as a build input", from) + } + if !strings.Contains(err.Error(), "outside its own repository") { + t.Fatalf("refused for the wrong reason: %v", err) + } + } +} + +func TestAnArtifactOfAnUnknownKindIsRefused(t *testing.T) { + _, err := ParseManifest([]byte(`{"module":"a","version":"1","build":{"artifacts":[ + {"name":"x","kind":"binary","from":"main.go"}]}}`)) + if err == nil { + t.Fatal("an artifact of an unknown kind was accepted") + } +} diff --git a/internal/catalogue/manifest.go b/internal/catalogue/manifest.go index ccba53a..0539ad9 100644 --- a/internal/catalogue/manifest.go +++ b/internal/catalogue/manifest.go @@ -171,6 +171,14 @@ type Manifest struct { // secret is the one thing that must not be. Serves map[string]map[string]any `json:"serves,omitempty"` + // Build says how this module's artifacts are produced from its source. + // + // The manifest in a repository names artifacts; the manifest the mesh holds names digests. + // **They are not the same document**, and that is deliberate: a digest is not knowable until + // something is built, and a repository that carried one would be a repository whose file is + // wrong the moment anybody edits anything. + Build *Build `json:"build,omitempty"` + // Binds is where this module wants to be told about something it requires, per requirement. // // Because "this machine needs a database from the anchor" is useless to the program that @@ -199,6 +207,34 @@ type Manifest struct { Grants map[string]string `json:"grants,omitempty"` } +// Build says how to produce this module's artifacts from its source. +// +// **Absent means nothing is built.** A module can be entirely configuration — a shell's rc file, +// a set of firewall rules — and having to declare an empty build for it would be a field that +// exists to be left blank. +type Build struct { + // Artifacts are what the source produces, each named so a resource can refer to it before + // anybody knows its digest. + Artifacts []Artifact `json:"artifacts,omitempty"` +} + +// Artifact is one thing built from a module's source. +type Artifact struct { + // Name is how resources refer to it. Local to the module. + Name string `json:"name"` + // Kind is "image" or "archive". + Kind string `json:"kind"` + // From is what it is built from, relative to the repository root: a Dockerfile for an image, + // a directory for an archive. + From string `json:"from"` +} + +// Kinds an artifact may be. +const ( + ArtifactImage = "image" + ArtifactArchive = "archive" +) + // SecretID is the resource identity of the file a module is given a credential in. func SecretID(requirement string) string { return "secret-" + requirement } @@ -307,6 +343,7 @@ func ParseManifest(raw []byte) (Manifest, error) { "%s contributes nothing to %q; if it only needs one, require it", m.Module, to)) } } + problems = append(problems, m.Build.problems(m.Module)...) for to := range m.Serves { var offered bool for _, o := range m.Offers() {