novox/mesh-controller
8
0
Fork 0
Code Issues Pull Requests 5 Releases Wiki Activity

Let an archive be an upstream release pinned by URL and sha256, mirrored by the build seat (hq issue 345, ADR 0316)

Browse Source 
A mail client's release is about 80 MB per version, too large to keep in a
repository at every upgrade, and an add-on whose licence forbids copying it
to third parties may not be in a repository at all. An archive artifact may
now name url, sha256 and version (and file or within) instead of from: the
build seat fetches it once, refuses it when its digest is not the pin's and
names both, keeps it in its workspace under the digest (a vendor that serves
only its newest version cannot break a later build of the same pin), and
packs it like any archive, so a node gets the mesh's own archive by digest
and never asks the vendor. The build seat's image carries xz, which Go's
standard library does not read. The installation check takes the vendor's
address and version of such a pin for the world's, as a registry is an
image's.
This commit is contained in:
jochen
2026-10-12 13:51:35 +02:00
parent 4b1dee6b79
commit b849af53dd
8 changed files with 786 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/catalogue/build.go
+59 -1
View File
@@ -2,7 +2,9 @@ package catalogue
import (
"fmt"
"net/url"
"path"
"regexp"
"sort"
"strings"
)
@@ -312,7 +314,7 @@ func (b *Build) problems(module string) []string {
module, a.Name, a.System, spokenSystems()))
}
} else {
if a.From == "" {
if a.From == "" && a.URL == "" {
problems = append(problems, fmt.Sprintf(
"%s: %q says nothing about what it is built from", module, a.Name))
}
@@ -350,6 +352,62 @@ func (b *Build) problems(module string) []string {
problems = append(problems, fmt.Sprintf(
"%s: %q mirrors %q, which names no tag or digest", module, a.Name, a.From))
}
problems = append(problems, upstreamFileProblems(module, a)...)
}
return problems
}
// sha256Hex is a pinned sha256 as a manifest writes it: 64 lower-case hexadecimal characters.
var sha256Hex = regexp.MustCompile(`^[0-9a-f]{64}$`)
// upstreamFileProblems is what is wrong with an archive pinned by URL (novox/hq ADR 0316 §2, §6): it is an
// archive, fetched over https, pinned by a sha256 and a version, and it names no directory of the repository
// besides; a file name or a directory within is a plain path that stays where it is put.
func upstreamFileProblems(module string, a Artifact) []string {
pinned := a.URL != "" || a.SHA256 != "" || a.Version != "" || a.File != "" || a.Within != ""
if !pinned {
return nil
}
var problems []string
say := func(format string, args ...any) {
problems = append(problems, fmt.Sprintf("%s: %q ", module, a.Name)+fmt.Sprintf(format, args...))
}
if a.Kind != ArtifactArchive {
say("is a %q and names an upstream file (url, sha256, version, file, within): only an archive is "+
"mirrored from one (novox/hq ADR 0316)", a.Kind)
return problems
}
if a.URL == "" {
say("names a sha256, a version, a file or a directory within, and no url to fetch it from")
return problems
}
if a.From != "" {
say("is built from %q and fetched from %s: an archive is one or the other", a.From, a.URL)
}
if u, err := url.Parse(a.URL); err != nil || u.Scheme != "https" || u.Host == "" {
say("is fetched from %q, and an upstream file is fetched over https from a named host", a.URL)
}
if !sha256Hex.MatchString(a.SHA256) {
say("is fetched from %s without a pinned sha256 (64 lower-case hexadecimal characters): what that "+
"address serves changes under the mesh, and only the digest says it is what was reviewed", a.URL)
}
if strings.TrimSpace(a.Version) == "" {
say("pins %s and says no version, so a person reading the pin cannot tell which release it is", a.URL)
}
if a.File != "" && a.Within != "" {
say("names a file (%q) and a directory within (%q): a file is kept whole, a directory is taken out "+
"of an archive", a.File, a.Within)
}
for field, p := range map[string]string{"file": a.File, "within": a.Within} {
if p == "" {
continue
}
clean := path.Clean(p)
if clean != p || strings.HasPrefix(p, "/") || clean == "." || clean == ".." || strings.HasPrefix(clean, "../") ||
(field == "file" && strings.Contains(p, "/")) {
say("names %s %q, which is not a plain relative %s", field, p, map[string]string{
"file": "file name", "within": "directory"}[field])
}
}
return problems
}
internal/catalogue/installation.go
+9
View File
@@ -71,6 +71,10 @@ var hostname = regexp.MustCompile(
`(?:be|nl|de|fr|uk|eu|com|net|org|io|dev|app|cloud|site|online|me|co|ch|at|lu|` +
`internal|example|tld|test|invalid)`)
// upstreamPin is where an artifact pinned by URL says where it is published and which release it is: the
// vendor's address and the vendor's version, which name no installation (Artifact.URL).
var upstreamPin = regexp.MustCompile(`^build\.artifacts\[[0-9]+\]\.(url|version)$`)
// address is a dotted quad.
var address = regexp.MustCompile(`(?:[0-9]{1,3}\.){3}[0-9]{1,3}`)
@@ -113,6 +117,11 @@ func InstallationProblems(m Manifest) []string {
"%s is named after %s, and a module is named for what it is, not for where it runs", m.Module, name))
}
walk(tree, "", nil, func(at string, value string, meant map[string]bool, isImage bool) {
// An archive pinned by URL names the vendor that publishes it, and the version it pins, which may
// have four dotted numbers (novox/hq ADR 0316): the world's, as a public registry is an image's.
if upstreamPin.MatchString(at) {
return
}
for _, name := range namesIn(value) {
if (isImage && worldsRegistries[strings.ToLower(name)]) || meant[name] {
continue
internal/catalogue/manifest.go
+26
View File
@@ -976,6 +976,32 @@ type Artifact struct {
// success, and was invisible to the launcher (novox/hq 04-ISSUES/142).
Binary string `json:"binary,omitempty"`
// URL is where an archive's upstream file is fetched from, for an archive that is a release
// somebody else published rather than a directory of this repository (novox/hq ADR 0316 §2, §6).
// With it come SHA256 and Version, and From stays empty.
//
// **Pinned by the digest of what the vendor published, mirrored by the build seat.** A release of
// about 80 MB copied into a repository is kept for ever in its history at every upgrade, and an
// add-on whose licence forbids copying it to third parties may not be in a repository at all. So the
// manifest names the file, the build seat fetches it, refuses it when its sha256 is not the pin's and
// names both digests, and packs it as any archive is packed: what reaches a machine is the mesh's own
// archive in the artifact store, fetched by digest like every other, and a machine never asks the
// vendor. Upgrading is changing the pin, reviewed like any other change.
URL string `json:"url,omitempty"`
// SHA256 is the sha256 of the file at URL, as 64 lower-case hexadecimal characters.
SHA256 string `json:"sha256,omitempty"`
// Version is what the vendor calls the release pinned, said where a person reads the pin; the build
// does not read it.
Version string `json:"version,omitempty"`
// File is the name an upstream file that is not an archive is kept under: the archive built from it
// holds that one file (an add-on's xpi). Empty means the file is a tar archive (gzip, bzip2 or xz)
// whose files are the artifact.
File string `json:"file,omitempty"`
// Within is the directory inside an upstream tar archive whose content is the artifact, for a
// release that unpacks into a directory of its own name (`betterbird/`). An entry outside it refuses
// the build. Empty means the archive's root.
Within string `json:"within,omitempty"`
// Entrypoints are the compiled files a tool host should load from this module, relative to the
// bundle's root.
//
internal/catalogue/upstream_file_test.go
+82
View File
@@ -0,0 +1,82 @@
package catalogue
import (
"strings"
"testing"
)
// An archive pinned by URL and sha256 (novox/hq ADR 0316 §2, §6): the vendor's file, mirrored by the build
// seat, never a copy in the repository.
const pinnedSum = "38dd24890a66b4cf9a45bc24e15192e00c26b929e1bb8df7974c16a4539129e2"
func upstreamManifest(artifact string) string {
return `{"module":"a","version":"1","build":{"artifacts":[` + artifact + `]}}`
}
func TestAnArchivePinnedByURLAndSha256IsAccepted(t *testing.T) {
for _, a := range []string{
`{"name":"archive","kind":"archive","url":"https://vendor.invalid/release-1.tar.xz","sha256":"` + pinnedSum +
`","version":"1.0","within":"release"}`,
`{"name":"addon","kind":"archive","url":"https://vendor.invalid/addon.xpi","sha256":"` + pinnedSum +
`","version":"1.5.6.1","file":"addon.xpi"}`,
} {
if _, err := ParseManifest([]byte(upstreamManifest(a))); err != nil {
t.Errorf("%s was refused: %v", a, err)
}
}
}
func TestAnUpstreamFileThatIsNotPinnedIsRefused(t *testing.T) {
cases := map[string]string{
// No digest: what the address serves changes under the mesh.
`{"name":"x","kind":"archive","url":"https://vendor.invalid/r.tar.xz","version":"1"}`: "without a pinned sha256",
`{"name":"x","kind":"archive","url":"https://vendor.invalid/r.tar.xz","sha256":"ABC","version":"1"}`: "without a pinned sha256",
// No version: a person reading the pin cannot tell which release.
`{"name":"x","kind":"archive","url":"https://vendor.invalid/r.tar.xz","sha256":"` + pinnedSum + `"}`: "says no version",
// Not https.
`{"name":"x","kind":"archive","url":"http://vendor.invalid/r.tar.xz","sha256":"` + pinnedSum + `","version":"1"}`: "over https",
// Both a directory of the repository and a URL.
`{"name":"x","kind":"archive","from":"files","url":"https://vendor.invalid/r.tar.xz","sha256":"` + pinnedSum +
`","version":"1"}`: "one or the other",
// A pin without a URL.
`{"name":"x","kind":"archive","from":"files","sha256":"` + pinnedSum + `"}`: "no url",
// Only an archive is mirrored from a file.
`{"name":"x","kind":"image","from":"Dockerfile","url":"https://vendor.invalid/r.tar.xz","sha256":"` + pinnedSum +
`","version":"1"}`: "only an archive",
// A file name that is a path, a directory within that leaves the archive, and both at once.
`{"name":"x","kind":"archive","url":"https://vendor.invalid/a.xpi","sha256":"` + pinnedSum +
`","version":"1","file":"../a.xpi"}`: "not a plain relative file name",
`{"name":"x","kind":"archive","url":"https://vendor.invalid/r.tar.xz","sha256":"` + pinnedSum +
`","version":"1","within":"../up"}`: "not a plain relative directory",
`{"name":"x","kind":"archive","url":"https://vendor.invalid/r.tar.xz","sha256":"` + pinnedSum +
`","version":"1","within":"r","file":"r.tar.xz"}`: "a file is kept whole",
}
for artifact, want := range cases {
_, err := ParseManifest([]byte(upstreamManifest(artifact)))
if err == nil {
t.Errorf("%s was accepted", artifact)
continue
}
if !strings.Contains(err.Error(), want) {
t.Errorf("%s was refused for another reason than %q: %v", artifact, want, err)
}
}
}
func TestAnUpstreamFilesVendorAndVersionNameNoInstallation(t *testing.T) {
// The vendor's address is the world's, as a public registry is an image's; a version of four dotted numbers
// is not an address. Anything else still is judged.
m, err := ParseManifest([]byte(upstreamManifest(`{"name":"owl","kind":"archive","url":"https://vendor.be/owl/owl.xpi","sha256":"` +
pinnedSum + `","version":"1.5.6.1","file":"owl.xpi"}`)))
if err != nil {
t.Fatal(err)
}
if problems := InstallationProblems(m); len(problems) != 0 {
t.Fatalf("the vendor and the version were taken for an installation: %v", problems)
}
m.Build.Artifacts[0].File = "mail.novox.be"
if problems := InstallationProblems(m); len(problems) != 1 || !strings.Contains(problems[0], "mail.novox.be") {
t.Fatalf("a name elsewhere in the artifact is still judged: %v", problems)
}
}