Merge pull request 'Let a declared file travel by its sha256 digest instead of its content, so a node's declaration is no longer bounded by one bus message (issue 1694, ADR 0486 step 1)' (#59) from fix/1694-file-references into main

go/catalogue/file_references_test.go

@@ -0,0 +1,10 @@
+package catalogue
+
+import "testing"
+
+// 1694/AC5: the capability a node-engine reports for files by digest (novox/hq ADR 0486 §5) is a known one.
+func TestFileReferencesIsAKnownCapability(t *testing.T) {
+	if !oneOf(KnownCapabilities, "file-references") {
+		t.Fatal("file-references is not a known capability")
+	}
+}

go/catalogue/seat_data.go

@@ -66,7 +66,7 @@ var capabilityName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`)
 // names an `if-capability` may give (novox/hq ADR 0255). A name nothing detects would leave its
 // contribution out on every machine, silently, so it is refused. A detector added there is added here.
 var KnownCapabilities = []string{
-	"battery", "container-runtime", "firewall", "graphical-session", "overlay", "package-manager",
+	"battery", "container-runtime", "file-references", "firewall", "graphical-session", "overlay", "package-manager",
 	"power-meter", "privileged", "seat", "service-manager", "uplink-dhcpcd", "uplink-networkmanager",
 	"uplink-systemd-networkd", "user-namespaces", "virtualisation",
 }

go/declaration/declaration.go

@@ -324,6 +324,15 @@ type File struct {
 	// depend on a field somewhere else.
 	Bytes string `json:"bytes,omitempty"`
 
+	// Digest and Size say the content is not carried but fetched: a file over FileReferenceThreshold travels as
+	// the sha256 digest of its content, "sha256:<64 hex>", and its size in bytes, in place of Content or Bytes
+	// (novox/hq ADR 0486 §1). The node-engine fetches the object of that name from FileBucket, checks what it
+	// received against Digest and Size, and only then writes the file. Exclusive with Content, Sealed, Bytes,
+	// VaultSealed and Secrets; sent only to an engine that reports FileReferencesCapability, as an older engine
+	// refuses a field it does not know, whole.
+	Digest string `json:"digest,omitempty"`
+	Size   int64  `json:"size,omitempty"`
+
 	// Owner is the user this belongs to, by name. Absent means root.
 	Owner string `json:"owner,omitempty"`
 
@@ -465,6 +474,10 @@ func (f *File) validate(where string, _ bool) []string {
 			problems = append(problems, where+": a file the vault sealed whole has no holes for secrets")
 		}
 	}
+	problems = append(problems, f.checkReference(where)...)
+	if f.Digest != "" || f.Size != 0 {
+		said = append(said, "digest")
+	}
 	if len(said) > 1 {
 		sort.Strings(said)
 		problems = append(problems, where+

go/declaration/filereference.go

@@ -0,0 +1,125 @@
+package declaration
+
+import (
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"fmt"
+	"regexp"
+)
+
+// The three names of a file travelling by digest (novox/hq ADR 0486 §1, §2, §5), shared by the node-engine that
+// reads them and the controller that writes them.
+const (
+	// FileReferenceThreshold is the size above which a file travels by digest: a file of this many bytes or fewer
+	// stays in the declaration as it is.
+	FileReferenceThreshold = 64 * 1024
+	// FileBucket is the object bucket of the bus that holds the contents, each under the hex of its sha256 digest
+	// (ObjectName). The controller alone writes it; a node-engine reads an object by name.
+	FileBucket = "declaration-files"
+	// FileReferencesCapability is what a node-engine reports when it reads a file given as a digest and size.
+	FileReferencesCapability = "file-references"
+)
+
+var digestForm = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`)
+
+// DigestOf is the digest a reference names for a content: "sha256:<64 lower-case hex>".
+func DigestOf(content []byte) string {
+	sum := sha256.Sum256(content)
+	return "sha256:" + hex.EncodeToString(sum[:])
+}
+
+// ObjectName is the name of the object holding the content a digest names: the hex without its "sha256:" prefix.
+func ObjectName(digest string) (string, error) {
+	if !digestForm.MatchString(digest) {
+		return "", fmt.Errorf("digest %q is not sha256:<64 lower-case hex characters>", digest)
+	}
+	return digest[len("sha256:"):], nil
+}
+
+// IsReference reports whether the file gives its content as a digest and size instead of carrying it.
+func (f *File) IsReference() bool { return f.Digest != "" }
+
+// checkReference judges the digest and size, and what a reference may not be given with.
+func (f *File) checkReference(where string) []string {
+	if f.Digest == "" && f.Size == 0 {
+		return nil
+	}
+	var problems []string
+	if !digestForm.MatchString(f.Digest) {
+		problems = append(problems, fmt.Sprintf("%s: digest %q; a file by reference names its content as "+
+			"sha256:<64 lower-case hex characters>", where, f.Digest))
+	}
+	if f.Size <= 0 {
+		problems = append(problems, fmt.Sprintf("%s: size %d; a file by reference states its size in bytes, "+
+			"more than zero", where, f.Size))
+	}
+	if f.Content != "" || f.Bytes != "" || f.Sealed != "" || f.VaultSealed != nil || len(f.Secrets) > 0 {
+		problems = append(problems, where+": a file by reference carries its content in the object its digest "+
+			"names, so it says no content, bytes, sealed, vault-sealed or secrets")
+	}
+	if f.Into != "" {
+		problems = append(problems, where+": a file by reference is written whole; a file written into says its "+
+			"part in content")
+	}
+	return problems
+}
+
+// Body is the bytes the file's content stands for: its content, or its bytes decoded.
+func (f *File) Body() ([]byte, error) {
+	if f.Bytes == "" {
+		return []byte(f.Content), nil
+	}
+	b, err := base64.StdEncoding.DecodeString(f.Bytes)
+	if err != nil {
+		return nil, fmt.Errorf("file %s: bytes are not base64: %w", f.ID, err)
+	}
+	return b, nil
+}
+
+// TravelsByDigest reports whether the controller may put this file's content in the bucket: it is plain content
+// or bytes (not sealed, not filled with secrets, not written into another file) of more than FileReferenceThreshold
+// bytes.
+func (f *File) TravelsByDigest() bool {
+	if f.IsReference() || f.Sealed != "" || f.VaultSealed != nil || len(f.Secrets) > 0 || f.Into != "" {
+		return false
+	}
+	if f.Bytes != "" {
+		// DecodedLen counts padding, so the exact length is taken off the decoded bytes.
+		body, err := f.Body()
+		return err == nil && len(body) > FileReferenceThreshold
+	}
+	return len(f.Content) > FileReferenceThreshold
+}
+
+// AsReference is the file as it is sent to an engine that reports FileReferencesCapability: a copy with the digest
+// and size in place of the content, and the content the controller puts into the bucket under ObjectName(digest).
+// A file that does not travel by digest is refused, not passed through, so a caller cannot send a small file empty.
+func (f *File) AsReference() (File, []byte, error) {
+	if !f.TravelsByDigest() {
+		return File{}, nil, fmt.Errorf("file %s does not travel by digest", f.ID)
+	}
+	body, err := f.Body()
+	if err != nil {
+		return File{}, nil, err
+	}
+	ref := *f
+	ref.Content, ref.Bytes = "", ""
+	ref.Digest, ref.Size = DigestOf(body), int64(len(body))
+	return ref, body, nil
+}
+
+// Verify checks what was fetched against the digest and size the file names; the node-engine writes the file only
+// after it returns nil. A mismatch is the apply's failure, never a partial apply (ADR 0486 §1).
+func (f *File) Verify(body []byte) error {
+	if !f.IsReference() {
+		return fmt.Errorf("file %s is not given by reference", f.ID)
+	}
+	if int64(len(body)) != f.Size {
+		return fmt.Errorf("file %s: the object is %d bytes and the declaration says %d", f.ID, len(body), f.Size)
+	}
+	if got := DigestOf(body); got != f.Digest {
+		return fmt.Errorf("file %s: the object hashes to %s and the declaration names %s", f.ID, got, f.Digest)
+	}
+	return nil
+}

go/declaration/filereference_test.go

@@ -0,0 +1,136 @@
+package declaration
+
+import (
+	"bytes"
+	"encoding/base64"
+	"encoding/json"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func refFile(extra string) string {
+	return `{"declaration":1,"resources":[{"id":"m.f","type":"file","path":"/etc/m","digest":"sha256:` +
+		strings.Repeat("a", 64) + `","size":70000` + extra + `}]}`
+}
+
+// 1694/AC1: a reference round trips, and an invalid one is refused naming the field.
+func TestAFileReferenceRoundTripsAndABadOneNamesItsField(t *testing.T) {
+	d, err := Parse([]byte(refFile("")))
+	if err != nil {
+		t.Fatal(err)
+	}
+	f := d.Resources[0].(*File)
+	if !f.IsReference() || f.Size != 70000 || f.Digest != "sha256:"+strings.Repeat("a", 64) {
+		t.Fatalf("read: %+v", f)
+	}
+	raw, _ := json.Marshal(d.Resources[0])
+	d2, err := Parse([]byte(`{"declaration":1,"resources":[` + string(raw) + `]}`))
+	if err != nil || !reflect.DeepEqual(d2.Resources[0], d.Resources[0]) {
+		t.Fatalf("round trip: %v %s", err, raw)
+	}
+	for want, raw := range map[string]string{
+		"digest":     strings.Replace(refFile(""), strings.Repeat("a", 64), "abc", 1),
+		"size":       strings.Replace(refFile(""), `"size":70000`, `"size":0`, 1),
+		"lower-case": strings.Replace(refFile(""), strings.Repeat("a", 64), strings.Repeat("A", 64), 1),
+		"content":    refFile(`,"content":"x"`),
+		"bytes":      refFile(`,"bytes":"eA=="`),
+		"whole":      refFile(`,"into":"block"`),
+	} {
+		_, err := Parse([]byte(raw))
+		if err == nil || !strings.Contains(err.Error(), want) {
+			t.Errorf("want a refusal naming %q, got %v", want, err)
+		}
+	}
+	// a size alone is not a reference
+	if _, err := Parse([]byte(`{"declaration":1,"resources":[{"id":"m.f","type":"file","path":"/e","size":5}]}`)); err == nil {
+		t.Error("a size without a digest was accepted")
+	}
+}
+
+// 1694/AC2: a declaration as today's SDK writes it reads unchanged and its bytes carry no new key.
+func TestAnInlineFileIsReadAndWrittenAsBefore(t *testing.T) {
+	const in = `{"id":"m.f","type":"file","path":"/e","content":"hello"}`
+	d, err := Parse([]byte(`{"declaration":1,"resources":[` + in + `]}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	out, _ := json.Marshal(d.Resources[0])
+	if string(out) != in {
+		t.Fatalf("signed bytes changed: %s", out)
+	}
+	if d.Resources[0].(*File).IsReference() {
+		t.Fatal("inline read as a reference")
+	}
+}
+
+// 1694/AC3: the controller's helper makes a reference of a big file and the engine's check refuses a changed byte.
+func TestABigFileBecomesAReferenceAndAChangedByteFailsTheCheck(t *testing.T) {
+	big := strings.Repeat("x", FileReferenceThreshold+1)
+	f := &File{ID: "m.f", Type: TypeFile, Path: "/e", Content: big}
+	if !f.TravelsByDigest() {
+		t.Fatal("a file over the threshold stays inline")
+	}
+	ref, body, err := f.AsReference()
+	if err != nil || ref.Content != "" || ref.Size != int64(len(big)) || string(body) != big {
+		t.Fatalf("%v %+v", err, ref)
+	}
+	if p := ref.validate("f", false); len(p) > 0 {
+		t.Fatalf("the reference is refused: %v", p)
+	}
+	if name, err := ObjectName(ref.Digest); err != nil || len(name) != 64 {
+		t.Fatalf("object name %q %v", name, err)
+	}
+	if err := ref.Verify(body); err != nil {
+		t.Fatal(err)
+	}
+	bad := bytes.Clone(body)
+	bad[7] ^= 1
+	if err := ref.Verify(bad); err == nil || !strings.Contains(err.Error(), "hashes to") {
+		t.Fatalf("a changed byte: %v", err)
+	}
+	if err := ref.Verify(body[1:]); err == nil || !strings.Contains(err.Error(), "bytes") {
+		t.Fatalf("a short object: %v", err)
+	}
+}
+
+// 1694/AC4: what stays inline, and bytes files travel decoded.
+func TestWhichFilesTravelByDigest(t *testing.T) {
+	small := strings.Repeat("x", FileReferenceThreshold)
+	big := strings.Repeat("x", FileReferenceThreshold+1)
+	for name, f := range map[string]File{
+		"at the threshold": {Content: small},
+		"sealed":           {Sealed: "s"},
+		"secrets":          {Content: big, Secrets: map[string]string{"a": "b"}},
+		"into":             {Content: big, Into: IntoBlock},
+	} {
+		if f.TravelsByDigest() {
+			t.Errorf("%s travels by digest", name)
+		}
+		if _, _, err := f.AsReference(); err == nil {
+			t.Errorf("%s was made a reference", name)
+		}
+	}
+	exact := File{ID: "m.e", Bytes: base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{7}, FileReferenceThreshold))}
+	if exact.TravelsByDigest() {
+		t.Error("a bytes file of exactly the threshold travels by digest")
+	}
+	if _, _, err := exact.AsReference(); err == nil {
+		t.Error("a bytes file of exactly the threshold was made a reference")
+	}
+	raw := bytes.Repeat([]byte{0, 1, 2}, FileReferenceThreshold)
+	f := File{ID: "m.b", Bytes: base64.StdEncoding.EncodeToString(raw)}
+	ref, body, err := f.AsReference()
+	if err != nil || !bytes.Equal(body, raw) || ref.Size != int64(len(raw)) || ref.Digest != DigestOf(raw) {
+		t.Fatalf("bytes file: %v", err)
+	}
+}
+
+func TestTheSharedNames(t *testing.T) {
+	if FileBucket != "declaration-files" || FileReferencesCapability != "file-references" || FileReferenceThreshold != 65536 {
+		t.Fatal("a name changed")
+	}
+	if _, err := ObjectName("sha256:zz"); err == nil {
+		t.Fatal("bad digest named an object")
+	}
+}