diff --git a/cmd/mesh-builder/Dockerfile b/cmd/mesh-builder/Dockerfile index 3951536d..57e93564 100644 --- a/cmd/mesh-builder/Dockerfile +++ b/cmd/mesh-builder/Dockerfile @@ -16,7 +16,8 @@ RUN CGO_ENABLED=0 go build -trimpath -ldflags '-s -w' -o /mesh-builder ./cmd/mes FROM ${ALPINE_BASE} # git to clone what it is asked to build, and the docker client to build and push it. The daemon # is the machine's, reached through its socket — a build machine shares the runtime it was given -# rather than running one inside itself. -RUN apk add --no-cache git docker-cli +# rather than running one inside itself. xz to read an upstream release an archive is pinned to +# (novox/hq ADR 0316): Go's standard library reads gzip and bzip2 and not xz. +RUN apk add --no-cache git docker-cli xz COPY --from=build /mesh-builder /usr/local/bin/mesh-builder ENTRYPOINT ["/usr/local/bin/mesh-builder"] diff --git a/internal/builder/builder.go b/internal/builder/builder.go index e142ef29..5ce8ae9c 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -971,8 +971,19 @@ func one(ctx context.Context, run Runner, publish Publisher, return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: reference}, nil case catalogue.ArtifactArchive: - src.ownHas(cleanEntry(a.From) + "/**") - body, err := pack(filepath.Join(tree, a.From)) + var body []byte + var err error + if a.URL != "" { + // An upstream file pinned by URL and sha256 (novox/hq ADR 0316): its build source is the pin, which + // is in module.json, always part of it. + body, err = upstreamArchive(ctx, workspace, a, say) + if err != nil { + return catalogue.Built{}, fmt.Errorf("%s: %s: %w", module, a.Name, err) + } + } else { + src.ownHas(cleanEntry(a.From) + "/**") + body, err = pack(filepath.Join(tree, a.From)) + } if err != nil { return catalogue.Built{}, fmt.Errorf("%s: packing %s failed: %w", module, a.Name, err) } diff --git a/internal/builder/upstream_file.go b/internal/builder/upstream_file.go new file mode 100644 index 00000000..d5ef52ee --- /dev/null +++ b/internal/builder/upstream_file.go @@ -0,0 +1,295 @@ +package builder + +import ( + "archive/tar" + "bufio" + "bytes" + "compress/bzip2" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path" + "path/filepath" + "strings" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// An archive whose source is somebody else's published file, pinned by URL and sha256 (novox/hq ADR 0316 +// §2, §6, to-be 56 §7): a mail client's release, an add-on's xpi. +// +// **Fetched by the build seat, checked against the pin before anything is read from it, and packed as any +// archive is packed.** What a machine receives is the mesh's own archive from the artifact store, by digest, +// exactly as for a directory of the repository; no machine ever asks the vendor. A file whose bytes are not +// the pin's fails the build and names both digests, because the address serving something else is the one +// thing the pin exists to catch. +// +// **Kept in the build seat's workspace under its sha256**, so it is fetched once: a vendor that serves only +// its newest version at one address (Owl) would otherwise break every later build of the same pin. + +// maxUpstream is the largest upstream file fetched, and the largest a release may unpack to: the node-engine +// unpacks no archive larger than this either (mesh-host internal/apply/archive.go, maxArchive). +const maxUpstream = 512 << 20 + +// upstreamClient fetches upstream files: a variable so a test serves its own. +var upstreamClient = &http.Client{Timeout: 15 * time.Minute} + +// upstreamDir is where the build seat keeps the upstream files it fetched, by sha256. +func upstreamDir(workspace string) string { return filepath.Join(workspace, "upstream") } + +// upstreamArchive is the packed archive of an artifact pinned by URL: the file fetched (or kept from an +// earlier build) and checked, then laid out as the artifact says and packed. +func upstreamArchive(ctx context.Context, workspace string, a catalogue.Artifact, + say func(step, format string, args ...any)) ([]byte, error) { + file, err := upstreamFile(ctx, workspace, a, say) + if err != nil { + return nil, err + } + laid := filepath.Join(workspace, "upstream-"+a.Name) + if err := os.RemoveAll(laid); err != nil { + return nil, err + } + defer os.RemoveAll(laid) + if err := os.MkdirAll(laid, 0o755); err != nil { + return nil, err + } + if a.File != "" { + body, err := os.ReadFile(file) + if err != nil { + return nil, err + } + if err := os.WriteFile(filepath.Join(laid, a.File), body, 0o644); err != nil { + return nil, err + } + } else if err := unpackUpstream(ctx, file, laid, a.Within); err != nil { + return nil, fmt.Errorf("%s (%s): %w", a.URL, a.Version, err) + } + return pack(laid) +} + +// upstreamFile is the path of the pinned file in the workspace: kept from an earlier build when its bytes +// are still the pin's, else fetched now and kept only when they are. +func upstreamFile(ctx context.Context, workspace string, a catalogue.Artifact, + say func(step, format string, args ...any)) (string, error) { + want := strings.ToLower(a.SHA256) + dir := upstreamDir(workspace) + kept := filepath.Join(dir, want) + if got, err := sumOf(kept); err == nil { + if got == want { + say("upstream", "%s %s: kept from an earlier build (sha256:%s)", a.Name, a.Version, short(want)) + return kept, nil + } + // Changed on disk since it was kept: fetched again rather than trusted. + os.Remove(kept) + } + if err := os.MkdirAll(dir, 0o755); err != nil { + return "", err + } + say("upstream", "fetching %s (%s)", a.URL, a.Version) + request, err := http.NewRequestWithContext(ctx, http.MethodGet, a.URL, nil) + if err != nil { + return "", err + } + response, err := upstreamClient.Do(request) + if err != nil { + return "", fmt.Errorf("cannot fetch %s: %w", a.URL, err) + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + return "", fmt.Errorf("%s answered %s", a.URL, response.Status) + } + part, err := os.CreateTemp(dir, want+".*.part") + if err != nil { + return "", err + } + defer os.Remove(part.Name()) + hash := sha256.New() + n, err := io.Copy(io.MultiWriter(part, hash), io.LimitReader(response.Body, maxUpstream+1)) + if cerr := part.Close(); err == nil { + err = cerr + } + if err != nil { + return "", fmt.Errorf("fetching %s: %w", a.URL, err) + } + if n > maxUpstream { + return "", fmt.Errorf("%s is larger than %d bytes, more than a machine unpacks; nothing was kept", a.URL, maxUpstream) + } + got := hex.EncodeToString(hash.Sum(nil)) + if got != want { + return "", fmt.Errorf("%s was pinned as sha256:%s and what it served is sha256:%s (%d bytes); nothing "+ + "was kept. The address serves another file than the one reviewed: compare it, then move the pin "+ + "in a reviewed change (novox/hq ADR 0316)", a.URL, want, got, n) + } + if err := os.Rename(part.Name(), kept); err != nil { + return "", err + } + say("upstream", "%s %s: %d bytes, sha256:%s as pinned", a.Name, a.Version, n, short(want)) + return kept, nil +} + +// sumOf is the sha256 of a file, in hexadecimal. +func sumOf(file string) (string, error) { + f, err := os.Open(file) + if err != nil { + return "", err + } + defer f.Close() + hash := sha256.New() + if _, err := io.Copy(hash, f); err != nil { + return "", err + } + return hex.EncodeToString(hash.Sum(nil)), nil +} + +// unpackUpstream unpacks a tar archive, compressed with gzip, bzip2 or xz (told by its first bytes), into a +// directory: of its entries only those within `within` (all, when empty), with that prefix taken off. +// +// **Only files and directories, and nothing outside.** A link, a device or an entry naming a path outside +// the archive, or outside `within`, refuses the build by name: the node-engine unpacks only files and +// directories, and an archive quietly unpacked without what it held would be a release that is not the one +// pinned. +func unpackUpstream(ctx context.Context, file, into, within string) error { + f, err := os.Open(file) + if err != nil { + return err + } + defer f.Close() + buffered := bufio.NewReader(f) + magic, _ := buffered.Peek(6) + var stream io.Reader + var wait func() error + switch { + case bytes.HasPrefix(magic, []byte{0x1f, 0x8b}): + zipped, err := gzip.NewReader(buffered) + if err != nil { + return err + } + defer zipped.Close() + stream = zipped + case bytes.HasPrefix(magic, []byte("BZh")): + stream = bzip2.NewReader(buffered) + case bytes.HasPrefix(magic, []byte{0xfd, '7', 'z', 'X', 'Z', 0x00}): + // Go's standard library reads no xz; the build seat's image carries xz (cmd/mesh-builder/Dockerfile). + if _, err := exec.LookPath("xz"); err != nil { + return errors.New("it is compressed with xz, and this build seat has no xz to read it") + } + cmd := exec.CommandContext(ctx, "xz", "-dc") + cmd.Stdin = buffered + var stderr bytes.Buffer + cmd.Stderr = &stderr + out, err := cmd.StdoutPipe() + if err != nil { + return err + } + if err := cmd.Start(); err != nil { + return err + } + stream = out + wait = func() error { + if err := cmd.Wait(); err != nil { + return fmt.Errorf("xz could not read it: %v %s", err, strings.TrimSpace(stderr.String())) + } + return nil + } + defer func() { + if wait != nil { + _ = cmd.Process.Kill() + _ = cmd.Wait() + } + }() + default: + return errors.New("it is not a tar archive compressed with gzip, bzip2 or xz; a single file is kept " + + "whole by naming it in `file`") + } + if err := untar(stream, into, within); err != nil { + return err + } + if wait != nil { + w := wait + wait = nil + return w() + } + return nil +} + +// untar writes a tar stream's files and directories within `within` into a directory. +func untar(stream io.Reader, into, within string) error { + reader := tar.NewReader(stream) + prefix := "" + if within != "" { + prefix = path.Clean(within) + "/" + } + var total int64 + files := 0 + for { + header, err := reader.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + name := path.Clean(strings.TrimPrefix(header.Name, "./")) + if path.IsAbs(name) || name == ".." || strings.HasPrefix(name, "../") { + return fmt.Errorf("%s names a path outside the archive", header.Name) + } + if name == "." || name+"/" == prefix { + continue + } + if prefix != "" { + if !strings.HasPrefix(name, prefix) { + return fmt.Errorf("%s is outside %s, the directory the artifact is taken from", header.Name, within) + } + name = strings.TrimPrefix(name, prefix) + } + target := filepath.Join(into, filepath.FromSlash(name)) + switch header.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(target, 0o755); err != nil { + return err + } + case tar.TypeReg: + total += header.Size + if total > maxUpstream { + return fmt.Errorf("it unpacks to more than %d bytes, more than a machine unpacks", maxUpstream) + } + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return err + } + mode := os.FileMode(0o644) + if header.Mode&0o111 != 0 { + mode = 0o755 + } + out, err := os.OpenFile(target, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, mode) + if err != nil { + return err + } + if _, err := io.Copy(out, io.LimitReader(reader, header.Size)); err != nil { + out.Close() + return err + } + if err := out.Close(); err != nil { + return err + } + files++ + default: + return fmt.Errorf("%s is a %c, and an archive carries only files and directories", header.Name, + header.Typeflag) + } + } + if files == 0 { + if within != "" { + return fmt.Errorf("it holds no file within %s", within) + } + return errors.New("it holds no file") + } + return nil +} diff --git a/internal/builder/upstream_file_test.go b/internal/builder/upstream_file_test.go new file mode 100644 index 00000000..6e233dce --- /dev/null +++ b/internal/builder/upstream_file_test.go @@ -0,0 +1,299 @@ +package builder + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "io" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// An archive pinned by URL and sha256 (novox/hq ADR 0316 §2, §6; How it is checked: "an upstream artifact +// whose download does not match its sha256 fails the build and names both digests"). + +type tarEntry struct { + name, body string + mode int64 + kind byte +} + +func tarOf(t *testing.T, entries []tarEntry) []byte { + t.Helper() + var raw bytes.Buffer + w := tar.NewWriter(&raw) + for _, e := range entries { + kind := e.kind + if kind == 0 { + kind = tar.TypeReg + } + h := &tar.Header{Name: e.name, Mode: e.mode, Typeflag: kind, Size: int64(len(e.body))} + if kind == tar.TypeSymlink { + h.Linkname, h.Size = "/usr/share/hunspell", 0 + } + if kind == tar.TypeDir { + h.Size = 0 + } + if err := w.WriteHeader(h); err != nil { + t.Fatal(err) + } + if kind == tar.TypeReg { + if _, err := w.Write([]byte(e.body)); err != nil { + t.Fatal(err) + } + } + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + return raw.Bytes() +} + +func gzipped(t *testing.T, body []byte) []byte { + t.Helper() + var out bytes.Buffer + z := gzip.NewWriter(&out) + if _, err := z.Write(body); err != nil { + t.Fatal(err) + } + if err := z.Close(); err != nil { + t.Fatal(err) + } + return out.Bytes() +} + +func sumHex(b []byte) string { + s := sha256.Sum256(b) + return hex.EncodeToString(s[:]) +} + +// vendor serves files by path over https and counts what it was asked. +type vendor struct { + files map[string][]byte + asked int +} + +func serveVendor(t *testing.T, v *vendor) string { + t.Helper() + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + v.asked++ + body, ok := v.files[r.URL.Path] + if !ok { + http.NotFound(w, r) + return + } + w.Write(body) + })) + t.Cleanup(server.Close) + before := upstreamClient + upstreamClient = server.Client() + t.Cleanup(func() { upstreamClient = before }) + return server.URL +} + +// entriesOf lists a packed archive: each file with its mode and content. +func entriesOf(t *testing.T, packed []byte) map[string]string { + t.Helper() + z, err := gzip.NewReader(bytes.NewReader(packed)) + if err != nil { + t.Fatal(err) + } + r := tar.NewReader(z) + out := map[string]string{} + for { + h, err := r.Next() + if err == io.EOF { + return out + } + if err != nil { + t.Fatal(err) + } + body, _ := io.ReadAll(r) + out[h.Name] = string(body) + " " + os.FileMode(h.Mode).String() + } +} + +func release(t *testing.T) []byte { + return gzipped(t, tarOf(t, []tarEntry{ + {name: "betterbird/", kind: tar.TypeDir, mode: 0o755}, + {name: "betterbird/application.ini", body: "[App]\nVersion=153.4.0\n", mode: 0o644}, + {name: "betterbird/betterbird", body: "#!/bin/sh\n", mode: 0o755}, + {name: "betterbird/defaults/pref/channel-prefs.js", body: "pref(\"app.update.channel\", \"default\");\n", mode: 0o644}, + })) +} + +func pinnedManifest(base, sum string) string { + return `{"module":"betterbird","version":"1", + "build":{"artifacts":[ + {"name":"archive","kind":"archive","url":"` + base + `/release.tar.gz","sha256":"` + sum + `","version":"153.4.0esr-bb10","within":"betterbird"}]}, + "resources":[{"id":"installation","type":"archive","path":"/opt/betterbird","artifact":"archive"}]}` +} + +func TestAnArchivePinnedByURLIsFetchedCheckedAndPackedWithoutItsOwnDirectory(t *testing.T) { + body := release(t) + v := &vendor{files: map[string][]byte{"/release.tar.gz": body}} + base := serveVendor(t, v) + r, workspace := aRepository(t, pinnedManifest(base, sumHex(body)), nil) + got, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil, + Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + digest, _ := got.Manifest.Resources[0]["digest"].(string) + if !strings.HasPrefix(digest, "sha256:") || r.archives["betterbird/archive"] != digest { + t.Fatalf("the release was not published and pinned as the module's archive: %v, published %v", + got.Manifest.Resources[0], r.archives) + } + + packed, err := upstreamArchive(context.Background(), workspace, catalogue.Artifact{Name: "archive", Kind: "archive", + URL: base + "/release.tar.gz", SHA256: sumHex(body), Version: "153.4.0esr-bb10", Within: "betterbird"}, + func(string, string, ...any) {}) + if err != nil { + t.Fatal(err) + } + if "sha256:"+sumHex(packed) != digest { + t.Fatalf("a second packing of one pin gave another digest: %s and %s", digest, sumHex(packed)) + } + entries := entriesOf(t, packed) + want := map[string]string{ + "application.ini": "[App]\nVersion=153.4.0\n -rw-r--r--", + "betterbird": "#!/bin/sh\n -rwxr-xr-x", + "defaults/pref/channel-prefs.js": "pref(\"app.update.channel\", \"default\");\n -rw-r--r--", + } + if len(entries) != len(want) { + t.Fatalf("the archive holds %v, want %v", entries, want) + } + for name, w := range want { + if entries[name] != w { + t.Errorf("%s is %q in the archive, want %q", name, entries[name], w) + } + } +} + +func TestAnUpstreamFileIsFetchedOnceAndKeptByItsDigest(t *testing.T) { + // The vendor of an add-on serves only its newest version at one address: a pin that was fetched once + // must still build after the address moved on. + body := release(t) + v := &vendor{files: map[string][]byte{"/release.tar.gz": body}} + base := serveVendor(t, v) + r, workspace := aRepository(t, pinnedManifest(base, sumHex(body)), nil) + first, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil, + Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + v.files["/release.tar.gz"] = []byte("the vendor's next release") + second, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil, + Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatalf("a pin fetched before failed to build once the address served something else: %v", err) + } + if v.asked != 1 { + t.Fatalf("the vendor was asked %d times for one pin, want once", v.asked) + } + if first.Manifest.Resources[0]["digest"] != second.Manifest.Resources[0]["digest"] { + t.Fatal("two builds of one pin gave two digests") + } +} + +func TestAnUpstreamFileWhoseDownloadDoesNotMatchItsPinFailsAndNamesBothDigests(t *testing.T) { + pinned := release(t) + served := gzipped(t, tarOf(t, []tarEntry{{name: "betterbird/betterbird", body: "not what was reviewed", mode: 0o755}})) + v := &vendor{files: map[string][]byte{"/release.tar.gz": served}} + base := serveVendor(t, v) + r, workspace := aRepository(t, pinnedManifest(base, sumHex(pinned)), nil) + _, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil, + Npmrc{}, GitCredential{}, nil) + if err == nil { + t.Fatal("a download that is not the pinned file was built") + } + for _, digest := range []string{sumHex(pinned), sumHex(served)} { + if !strings.Contains(err.Error(), "sha256:"+digest) { + t.Errorf("the refusal does not name sha256:%s: %v", digest, err) + } + } + if len(r.archives) != 0 { + t.Fatalf("something was published: %v", r.archives) + } + kept, _ := filepath.Glob(filepath.Join(upstreamDir(workspace), "*")) + if len(kept) != 0 { + t.Fatalf("the refused download was kept: %v", kept) + } +} + +func TestAnUpstreamFileThatIsNotAnArchiveIsKeptWholeUnderItsName(t *testing.T) { + xpi := []byte("PK\x03\x04 an add-on") + v := &vendor{files: map[string][]byte{"/owl.xpi": xpi}} + base := serveVendor(t, v) + packed, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "owl", Kind: "archive", + URL: base + "/owl.xpi", SHA256: sumHex(xpi), Version: "1.5.6.1", File: "owl.xpi"}, func(string, string, ...any) {}) + if err != nil { + t.Fatal(err) + } + entries := entriesOf(t, packed) + if len(entries) != 1 || entries["owl.xpi"] != string(xpi)+" -rw-r--r--" { + t.Fatalf("the add-on was not kept whole as owl.xpi: %v", entries) + } +} + +func TestAnUpstreamArchiveHoldingALinkOrAnEntryOutsideItsDirectoryIsRefused(t *testing.T) { + cases := map[string][]tarEntry{ + // The community package links the dictionaries; a release carrying a link is not one a machine unpacks. + "is a 2": {{name: "betterbird/betterbird", body: "x", mode: 0o755}, + {name: "betterbird/dictionaries", kind: tar.TypeSymlink}}, + "outside betterbird": {{name: "betterbird/betterbird", body: "x", mode: 0o755}, + {name: "README", body: "x", mode: 0o644}}, + "outside the archive": {{name: "../etc/passwd", body: "x", mode: 0o644}}, + "no file within": {{name: "betterbird/", kind: tar.TypeDir, mode: 0o755}}, + } + for want, entries := range cases { + body := gzipped(t, tarOf(t, entries)) + v := &vendor{files: map[string][]byte{"/r.tar.gz": body}} + base := serveVendor(t, v) + _, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "archive", Kind: "archive", + URL: base + "/r.tar.gz", SHA256: sumHex(body), Version: "1", Within: "betterbird"}, func(string, string, ...any) {}) + if err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("an archive whose entry is %q was not refused for it: %v", want, err) + } + } +} + +func TestAnUpstreamArchiveCompressedWithXzIsRead(t *testing.T) { + // Betterbird publishes its Linux release as .tar.xz; the build seat's image carries xz for it. + if _, err := exec.LookPath("xz"); err != nil { + t.Skip("NOT TESTED here: no xz on this machine (the build seat's image carries it)") + } + plain := tarOf(t, []tarEntry{{name: "betterbird/betterbird", body: "#!/bin/sh\n", mode: 0o755}}) + cmd := exec.Command("xz", "-zc") + cmd.Stdin = bytes.NewReader(plain) + compressed, err := cmd.Output() + if err != nil { + t.Fatal(err) + } + v := &vendor{files: map[string][]byte{"/r.tar.xz": compressed}} + base := serveVendor(t, v) + packed, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "archive", Kind: "archive", + URL: base + "/r.tar.xz", SHA256: sumHex(compressed), Version: "1", Within: "betterbird"}, func(string, string, ...any) {}) + if err != nil { + t.Fatal(err) + } + var names []string + for n := range entriesOf(t, packed) { + names = append(names, n) + } + sort.Strings(names) + if strings.Join(names, ",") != "betterbird" { + t.Fatalf("the xz release unpacked to %v", names) + } +} diff --git a/internal/catalogue/build.go b/internal/catalogue/build.go index 21f0e296..74815e39 100644 --- a/internal/catalogue/build.go +++ b/internal/catalogue/build.go @@ -2,7 +2,9 @@ package catalogue import ( "fmt" + "net/url" "path" + "regexp" "sort" "strings" ) @@ -312,7 +314,7 @@ func (b *Build) problems(module string) []string { module, a.Name, a.System, spokenSystems())) } } else { - if a.From == "" { + if a.From == "" && a.URL == "" { problems = append(problems, fmt.Sprintf( "%s: %q says nothing about what it is built from", module, a.Name)) } @@ -350,6 +352,62 @@ func (b *Build) problems(module string) []string { problems = append(problems, fmt.Sprintf( "%s: %q mirrors %q, which names no tag or digest", module, a.Name, a.From)) } + problems = append(problems, upstreamFileProblems(module, a)...) + } + return problems +} + +// sha256Hex is a pinned sha256 as a manifest writes it: 64 lower-case hexadecimal characters. +var sha256Hex = regexp.MustCompile(`^[0-9a-f]{64}$`) + +// upstreamFileProblems is what is wrong with an archive pinned by URL (novox/hq ADR 0316 §2, §6): it is an +// archive, fetched over https, pinned by a sha256 and a version, and it names no directory of the repository +// besides; a file name or a directory within is a plain path that stays where it is put. +func upstreamFileProblems(module string, a Artifact) []string { + pinned := a.URL != "" || a.SHA256 != "" || a.Version != "" || a.File != "" || a.Within != "" + if !pinned { + return nil + } + var problems []string + say := func(format string, args ...any) { + problems = append(problems, fmt.Sprintf("%s: %q ", module, a.Name)+fmt.Sprintf(format, args...)) + } + if a.Kind != ArtifactArchive { + say("is a %q and names an upstream file (url, sha256, version, file, within): only an archive is "+ + "mirrored from one (novox/hq ADR 0316)", a.Kind) + return problems + } + if a.URL == "" { + say("names a sha256, a version, a file or a directory within, and no url to fetch it from") + return problems + } + if a.From != "" { + say("is built from %q and fetched from %s: an archive is one or the other", a.From, a.URL) + } + if u, err := url.Parse(a.URL); err != nil || u.Scheme != "https" || u.Host == "" { + say("is fetched from %q, and an upstream file is fetched over https from a named host", a.URL) + } + if !sha256Hex.MatchString(a.SHA256) { + say("is fetched from %s without a pinned sha256 (64 lower-case hexadecimal characters): what that "+ + "address serves changes under the mesh, and only the digest says it is what was reviewed", a.URL) + } + if strings.TrimSpace(a.Version) == "" { + say("pins %s and says no version, so a person reading the pin cannot tell which release it is", a.URL) + } + if a.File != "" && a.Within != "" { + say("names a file (%q) and a directory within (%q): a file is kept whole, a directory is taken out "+ + "of an archive", a.File, a.Within) + } + for field, p := range map[string]string{"file": a.File, "within": a.Within} { + if p == "" { + continue + } + clean := path.Clean(p) + if clean != p || strings.HasPrefix(p, "/") || clean == "." || clean == ".." || strings.HasPrefix(clean, "../") || + (field == "file" && strings.Contains(p, "/")) { + say("names %s %q, which is not a plain relative %s", field, p, map[string]string{ + "file": "file name", "within": "directory"}[field]) + } } return problems } diff --git a/internal/catalogue/installation.go b/internal/catalogue/installation.go index d1325e72..94733e05 100644 --- a/internal/catalogue/installation.go +++ b/internal/catalogue/installation.go @@ -71,6 +71,10 @@ var hostname = regexp.MustCompile( `(?:be|nl|de|fr|uk|eu|com|net|org|io|dev|app|cloud|site|online|me|co|ch|at|lu|` + `internal|example|tld|test|invalid)`) +// upstreamPin is where an artifact pinned by URL says where it is published and which release it is: the +// vendor's address and the vendor's version, which name no installation (Artifact.URL). +var upstreamPin = regexp.MustCompile(`^build\.artifacts\[[0-9]+\]\.(url|version)$`) + // address is a dotted quad. var address = regexp.MustCompile(`(?:[0-9]{1,3}\.){3}[0-9]{1,3}`) @@ -113,6 +117,11 @@ func InstallationProblems(m Manifest) []string { "%s is named after %s, and a module is named for what it is, not for where it runs", m.Module, name)) } walk(tree, "", nil, func(at string, value string, meant map[string]bool, isImage bool) { + // An archive pinned by URL names the vendor that publishes it, and the version it pins, which may + // have four dotted numbers (novox/hq ADR 0316): the world's, as a public registry is an image's. + if upstreamPin.MatchString(at) { + return + } for _, name := range namesIn(value) { if (isImage && worldsRegistries[strings.ToLower(name)]) || meant[name] { continue diff --git a/internal/catalogue/manifest.go b/internal/catalogue/manifest.go index efb32aea..6ba3f51c 100644 --- a/internal/catalogue/manifest.go +++ b/internal/catalogue/manifest.go @@ -976,6 +976,32 @@ type Artifact struct { // success, and was invisible to the launcher (novox/hq 04-ISSUES/142). Binary string `json:"binary,omitempty"` + // URL is where an archive's upstream file is fetched from, for an archive that is a release + // somebody else published rather than a directory of this repository (novox/hq ADR 0316 §2, §6). + // With it come SHA256 and Version, and From stays empty. + // + // **Pinned by the digest of what the vendor published, mirrored by the build seat.** A release of + // about 80 MB copied into a repository is kept for ever in its history at every upgrade, and an + // add-on whose licence forbids copying it to third parties may not be in a repository at all. So the + // manifest names the file, the build seat fetches it, refuses it when its sha256 is not the pin's and + // names both digests, and packs it as any archive is packed: what reaches a machine is the mesh's own + // archive in the artifact store, fetched by digest like every other, and a machine never asks the + // vendor. Upgrading is changing the pin, reviewed like any other change. + URL string `json:"url,omitempty"` + // SHA256 is the sha256 of the file at URL, as 64 lower-case hexadecimal characters. + SHA256 string `json:"sha256,omitempty"` + // Version is what the vendor calls the release pinned, said where a person reads the pin; the build + // does not read it. + Version string `json:"version,omitempty"` + // File is the name an upstream file that is not an archive is kept under: the archive built from it + // holds that one file (an add-on's xpi). Empty means the file is a tar archive (gzip, bzip2 or xz) + // whose files are the artifact. + File string `json:"file,omitempty"` + // Within is the directory inside an upstream tar archive whose content is the artifact, for a + // release that unpacks into a directory of its own name (`betterbird/`). An entry outside it refuses + // the build. Empty means the archive's root. + Within string `json:"within,omitempty"` + // Entrypoints are the compiled files a tool host should load from this module, relative to the // bundle's root. // diff --git a/internal/catalogue/upstream_file_test.go b/internal/catalogue/upstream_file_test.go new file mode 100644 index 00000000..c4bbcda9 --- /dev/null +++ b/internal/catalogue/upstream_file_test.go @@ -0,0 +1,82 @@ +package catalogue + +import ( + "strings" + "testing" +) + +// An archive pinned by URL and sha256 (novox/hq ADR 0316 §2, §6): the vendor's file, mirrored by the build +// seat, never a copy in the repository. + +const pinnedSum = "38dd24890a66b4cf9a45bc24e15192e00c26b929e1bb8df7974c16a4539129e2" + +func upstreamManifest(artifact string) string { + return `{"module":"a","version":"1","build":{"artifacts":[` + artifact + `]}}` +} + +func TestAnArchivePinnedByURLAndSha256IsAccepted(t *testing.T) { + for _, a := range []string{ + `{"name":"archive","kind":"archive","url":"https://vendor.invalid/release-1.tar.xz","sha256":"` + pinnedSum + + `","version":"1.0","within":"release"}`, + `{"name":"addon","kind":"archive","url":"https://vendor.invalid/addon.xpi","sha256":"` + pinnedSum + + `","version":"1.5.6.1","file":"addon.xpi"}`, + } { + if _, err := ParseManifest([]byte(upstreamManifest(a))); err != nil { + t.Errorf("%s was refused: %v", a, err) + } + } +} + +func TestAnUpstreamFileThatIsNotPinnedIsRefused(t *testing.T) { + cases := map[string]string{ + // No digest: what the address serves changes under the mesh. + `{"name":"x","kind":"archive","url":"https://vendor.invalid/r.tar.xz","version":"1"}`: "without a pinned sha256", + `{"name":"x","kind":"archive","url":"https://vendor.invalid/r.tar.xz","sha256":"ABC","version":"1"}`: "without a pinned sha256", + // No version: a person reading the pin cannot tell which release. + `{"name":"x","kind":"archive","url":"https://vendor.invalid/r.tar.xz","sha256":"` + pinnedSum + `"}`: "says no version", + // Not https. + `{"name":"x","kind":"archive","url":"http://vendor.invalid/r.tar.xz","sha256":"` + pinnedSum + `","version":"1"}`: "over https", + // Both a directory of the repository and a URL. + `{"name":"x","kind":"archive","from":"files","url":"https://vendor.invalid/r.tar.xz","sha256":"` + pinnedSum + + `","version":"1"}`: "one or the other", + // A pin without a URL. + `{"name":"x","kind":"archive","from":"files","sha256":"` + pinnedSum + `"}`: "no url", + // Only an archive is mirrored from a file. + `{"name":"x","kind":"image","from":"Dockerfile","url":"https://vendor.invalid/r.tar.xz","sha256":"` + pinnedSum + + `","version":"1"}`: "only an archive", + // A file name that is a path, a directory within that leaves the archive, and both at once. + `{"name":"x","kind":"archive","url":"https://vendor.invalid/a.xpi","sha256":"` + pinnedSum + + `","version":"1","file":"../a.xpi"}`: "not a plain relative file name", + `{"name":"x","kind":"archive","url":"https://vendor.invalid/r.tar.xz","sha256":"` + pinnedSum + + `","version":"1","within":"../up"}`: "not a plain relative directory", + `{"name":"x","kind":"archive","url":"https://vendor.invalid/r.tar.xz","sha256":"` + pinnedSum + + `","version":"1","within":"r","file":"r.tar.xz"}`: "a file is kept whole", + } + for artifact, want := range cases { + _, err := ParseManifest([]byte(upstreamManifest(artifact))) + if err == nil { + t.Errorf("%s was accepted", artifact) + continue + } + if !strings.Contains(err.Error(), want) { + t.Errorf("%s was refused for another reason than %q: %v", artifact, want, err) + } + } +} + +func TestAnUpstreamFilesVendorAndVersionNameNoInstallation(t *testing.T) { + // The vendor's address is the world's, as a public registry is an image's; a version of four dotted numbers + // is not an address. Anything else still is judged. + m, err := ParseManifest([]byte(upstreamManifest(`{"name":"owl","kind":"archive","url":"https://vendor.be/owl/owl.xpi","sha256":"` + + pinnedSum + `","version":"1.5.6.1","file":"owl.xpi"}`))) + if err != nil { + t.Fatal(err) + } + if problems := InstallationProblems(m); len(problems) != 0 { + t.Fatalf("the vendor and the version were taken for an installation: %v", problems) + } + m.Build.Artifacts[0].File = "mail.novox.be" + if problems := InstallationProblems(m); len(problems) != 1 || !strings.Contains(problems[0], "mail.novox.be") { + t.Fatalf("a name elsewhere in the artifact is still judged: %v", problems) + } +}